agora inbox for pgsql-hackers@postgresql.orghelp / color / mirror / Atom feed
[PATCH v20 3/8] Row pattern recognition patch (rewriter). 324+ messages / 2 participants [nested] [flat]
* [PATCH v20 3/8] Row pattern recognition patch (rewriter). @ 2024-05-24 02:26 Tatsuo Ishii <ishii@postgresql.org> 0 siblings, 0 replies; 324+ messages in thread From: Tatsuo Ishii @ 2024-05-24 02:26 UTC (permalink / raw) --- src/backend/utils/adt/ruleutils.c | 103 ++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c index 9618619762..0863fad59d 100644 --- a/src/backend/utils/adt/ruleutils.c +++ b/src/backend/utils/adt/ruleutils.c @@ -426,6 +426,10 @@ static void get_rule_groupingset(GroupingSet *gset, List *targetlist, bool omit_parens, deparse_context *context); static void get_rule_orderby(List *orderList, List *targetList, bool force_colno, deparse_context *context); +static void get_rule_pattern(List *patternVariable, List *patternRegexp, + bool force_colno, deparse_context *context); +static void get_rule_define(List *defineClause, List *patternVariables, + bool force_colno, deparse_context *context); static void get_rule_windowclause(Query *query, deparse_context *context); static void get_rule_windowspec(WindowClause *wc, List *targetList, deparse_context *context); @@ -6460,6 +6464,67 @@ get_rule_orderby(List *orderList, List *targetList, } } +/* + * Display a PATTERN clause. + */ +static void +get_rule_pattern(List *patternVariable, List *patternRegexp, + bool force_colno, deparse_context *context) +{ + StringInfo buf = context->buf; + const char *sep; + ListCell *lc_var, + *lc_reg = list_head(patternRegexp); + + sep = ""; + appendStringInfoChar(buf, '('); + foreach(lc_var, patternVariable) + { + char *variable = strVal((String *) lfirst(lc_var)); + char *regexp = NULL; + + if (lc_reg != NULL) + { + regexp = strVal((String *) lfirst(lc_reg)); + + lc_reg = lnext(patternRegexp, lc_reg); + } + + appendStringInfo(buf, "%s%s", sep, variable); + if (regexp !=NULL) + appendStringInfoString(buf, regexp); + + sep = " "; + } + appendStringInfoChar(buf, ')'); +} + +/* + * Display a DEFINE clause. + */ +static void +get_rule_define(List *defineClause, List *patternVariables, + bool force_colno, deparse_context *context) +{ + StringInfo buf = context->buf; + const char *sep; + ListCell *lc_var, + *lc_def; + + sep = " "; + Assert(list_length(patternVariables) == list_length(defineClause)); + + forboth(lc_var, patternVariables, lc_def, defineClause) + { + char *varName = strVal(lfirst(lc_var)); + TargetEntry *te = (TargetEntry *) lfirst(lc_def); + + appendStringInfo(buf, "%s%s AS ", sep, varName); + get_rule_expr((Node *) te->expr, context, false); + sep = ",\n "; + } +} + /* * Display a WINDOW clause. * @@ -6597,6 +6662,44 @@ get_rule_windowspec(WindowClause *wc, List *targetList, appendStringInfoString(buf, "EXCLUDE GROUP "); else if (wc->frameOptions & FRAMEOPTION_EXCLUDE_TIES) appendStringInfoString(buf, "EXCLUDE TIES "); + /* RPR */ + if (wc->rpSkipTo == ST_NEXT_ROW) + appendStringInfoString(buf, + "\n AFTER MATCH SKIP TO NEXT ROW "); + else if (wc->rpSkipTo == ST_PAST_LAST_ROW) + appendStringInfoString(buf, + "\n AFTER MATCH SKIP PAST LAST ROW "); + else if (wc->rpSkipTo == ST_FIRST_VARIABLE) + appendStringInfo(buf, + "\n AFTER MATCH SKIP TO FIRST %s ", + wc->rpSkipVariable); + else if (wc->rpSkipTo == ST_LAST_VARIABLE) + appendStringInfo(buf, + "\n AFTER MATCH SKIP TO LAST %s ", + wc->rpSkipVariable); + else if (wc->rpSkipTo == ST_VARIABLE) + appendStringInfo(buf, + "\n AFTER MATCH SKIP TO %s ", + wc->rpSkipVariable); + + if (wc->initial) + appendStringInfoString(buf, "\n INITIAL"); + + if (wc->patternVariable) + { + appendStringInfoString(buf, "\n PATTERN "); + get_rule_pattern(wc->patternVariable, wc->patternRegexp, + false, context); + } + + if (wc->defineClause) + { + appendStringInfoString(buf, "\n DEFINE\n"); + get_rule_define(wc->defineClause, wc->patternVariable, + false, context); + appendStringInfoChar(buf, ' '); + } + /* we will now have a trailing space; remove it */ buf->len--; } -- 2.25.1 ----Next_Part(Fri_May_24_11_39_19_2024_763)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v20-0004-Row-pattern-recognition-patch-planner.patch" ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
* [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table @ 2025-07-15 10:45 Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 0 siblings, 0 replies; 324+ messages in thread From: Jehan-Guillaume de Rorthais @ 2025-07-15 10:45 UTC (permalink / raw) When a foreign table points to a partitioned table or an inheritance parent on the foreign server, a non-direct DML can affect multiple rows when only one row is intended to be affected. This happens because postgres_fdw uses only ctid to identify a row to work on. Though ctid uniquely identifies a row in a single table, in a partitioned table or in an inheritance hierarchy, there can be be multiple rows, in different partitions, with the same ctid. So DML statement sent to the foreign server by postgres_fdw ends up affecting more than one rows, only one of which is intended to be affected. This commit adds testcases to show the problem. A subsequent commit would have a fix to the problem. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Discussion: https://www.postgresql.org/message-id/flat/CAFjFpRfcgwsHRmpvoOK-GUQi-n8MgAS%2BOxcQo%3DaBDn1COywmcg%4... Rebased by Jehan-Guillaume de Rorthais <jgdr@dalibo.com> --- .../postgres_fdw/expected/postgres_fdw.out | 125 ++++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 58 ++++++++ 2 files changed, 183 insertions(+) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2185b42bb4f..62019eaa881 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8954,6 +8954,131 @@ drop foreign table remt2; drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | aaa + fa | (0,1) | bbb +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Update on public.fa + Remote SQL: UPDATE public.a SET aa = $2 WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: CASE WHEN (random() <= '1'::double precision) THEN 'zzzz'::text ELSE NULL::text END, ctid, fa.* + Remote SQL: SELECT aa, ctid FROM public.a WHERE ((aa = 'aaa')) FOR UPDATE +(5 rows) + +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+------ + fa | (0,2) | zzzz + fa | (0,1) | bbb +(2 rows) + +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Delete on public.fa + Remote SQL: DELETE FROM public.a WHERE ctid = $1 + -> Foreign Scan on public.fa + Output: ctid + Filter: (fa.aa = CASE WHEN (random() <= '1'::double precision) THEN 'aaa'::text ELSE 'bbb'::text END) + Remote SQL: SELECT aa, ctid FROM public.a FOR UPDATE +(6 rows) + +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + tableoid | ctid | aa +----------+-------+----- + fa | (0,1) | bbb +(1 row) + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; +NOTICE: drop cascades to table b +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 1 | 1 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Update on public.fplt + Remote SQL: UPDATE public.plt SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: CASE WHEN (random() <= '1'::double precision) THEN 10 ELSE 20 END, ctid, fplt.* + Remote SQL: SELECT a, b, ctid FROM public.plt WHERE ((a = 1)) FOR UPDATE +(5 rows) + +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+---- + fplt | (0,2) | 1 | 10 + fplt | (0,1) | 2 | 2 +(2 rows) + +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Delete on public.fplt + Remote SQL: DELETE FROM public.plt WHERE ctid = $1 + -> Foreign Scan on public.fplt + Output: ctid + Filter: (fplt.a = CASE WHEN (random() <= '1'::double precision) THEN 1 ELSE 10 END) + Remote SQL: SELECT a, ctid FROM public.plt FOR UPDATE +(6 rows) + +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + tableoid | ctid | a | b +----------+-------+---+--- + fplt | (0,1) | 2 | 2 +(1 row) + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index e534b40de3c..ae3777b7edb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2494,6 +2494,64 @@ drop table loct1; drop table loct2; drop table parent; +-- test DML statement on a foreign table pointing to an inheritance hierarchy +-- on the remote server +CREATE TABLE a(aa TEXT); +ALTER TABLE a SET (autovacuum_enabled = 'false'); +CREATE TABLE b() INHERITS(a); +ALTER TABLE b SET (autovacuum_enabled = 'false'); +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +CREATE FOREIGN TABLE fa (aa TEXT) SERVER loopback OPTIONS (table_name 'a'); + +SELECT tableoid::regclass, ctid, * FROM fa; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +UPDATE fa SET aa = (CASE WHEN random() <= 1 THEN 'zzzz' ELSE NULL END) WHERE aa = 'aaa'; +SELECT tableoid::regclass, ctid, * FROM fa; +-- repopulate tables so that we have rows with same ctid +TRUNCATE a, b; +INSERT INTO a(aa) VALUES('aaa'); +INSERT INTO b(aa) VALUES('bbb'); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +DELETE FROM fa WHERE aa = (CASE WHEN random() <= 1 THEN 'aaa' ELSE 'bbb' END); +SELECT tableoid::regclass, ctid, * FROM fa; + +-- cleanup +DROP FOREIGN TABLE fa; +DROP TABLE a CASCADE; + +-- =================================================================== +-- test foreign table pointing to a remote partitioned table +-- =================================================================== + +-- test DML statement on foreign table pointing to a foreign partitioned table +CREATE TABLE plt (a int, b int) PARTITION BY LIST(a); +CREATE TABLE plt_p1 PARTITION OF plt FOR VALUES IN (1); +CREATE TABLE plt_p2 PARTITION OF plt FOR VALUES IN (2); +INSERT INTO plt VALUES (1, 1), (2, 2); +CREATE FOREIGN TABLE fplt (a int, b int) SERVER loopback OPTIONS (table_name 'plt'); +SELECT tableoid::regclass, ctid, * FROM fplt; +-- use random() so that DML statement is not pushed down to the foreign +-- server +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +UPDATE fplt SET b = (CASE WHEN random() <= 1 THEN 10 ELSE 20 END) WHERE a = 1; +SELECT tableoid::regclass, ctid, * FROM fplt; +-- repopulate partitioned table so that we have rows with same ctid +TRUNCATE plt; +INSERT INTO plt VALUES (1, 1), (2, 2); +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +DELETE FROM fplt WHERE a = (CASE WHEN random() <= 1 THEN 1 ELSE 10 END); +SELECT tableoid::regclass, ctid, * FROM fplt; + +DROP TABLE plt; +DROP FOREIGN TABLE fplt; + -- =================================================================== -- test tuple routing for foreign-table partitions -- =================================================================== -- 2.50.0 --MP_/4ZRYdF7Ah.pt5w65PuZdaPz Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=v3-0002-Error-out-if-one-iteration-of-non-direct-DML-affe.patch ^ permalink raw reply [nested|flat] 324+ messages in thread
end of thread, other threads:[~2025-07-15 10:45 UTC | newest] Thread overview: 324+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2024-05-24 02:26 [PATCH v20 3/8] Row pattern recognition patch (rewriter). Tatsuo Ishii <ishii@postgresql.org> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com> 2025-07-15 10:45 [PATCH v3 1/2] Test exposing bug when foreign table points to a partitioned table Jehan-Guillaume de Rorthais <jgdr@dalibo.com>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox