public inbox for [email protected]help / color / mirror / Atom feed
[PATCH 09/10] Add zstd compression levels 19+ messages / 6 participants [nested] [flat]
* [PATCH 09/10] Add zstd compression levels @ 2021-03-14 22:12 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 19+ messages in thread From: Justin Pryzby @ 2021-03-14 22:12 UTC (permalink / raw) --- src/backend/access/transam/xlog.c | 6 +++++- src/backend/access/transam/xloginsert.c | 15 +++++++++++++-- src/backend/access/transam/xlogreader.c | 12 ++++++++++++ src/backend/utils/misc/guc.c | 2 +- src/include/access/xlog_internal.h | 4 ++++ 5 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 92023de9f5..b14c7c5929 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,7 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; -int wal_compression_method = WAL_COMPRESSION_ZSTD; +int wal_compression_method = WAL_COMPRESSION_ZSTD_FAST_10; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; @@ -192,6 +192,10 @@ const struct config_enum_entry wal_compression_options[] = { #endif #ifdef USE_ZSTD {"zstd", WAL_COMPRESSION_ZSTD, false}, + {"zstd-1", WAL_COMPRESSION_ZSTD_1, false}, + {"zstd-fast-10", WAL_COMPRESSION_ZSTD_FAST_10, false}, + {"zstd-fast-20", WAL_COMPRESSION_ZSTD_FAST_20, false}, + {"zstd-fast-40", WAL_COMPRESSION_ZSTD_FAST_40, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 4591e476c6..4f11f96373 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -915,11 +915,22 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, #ifdef USE_ZSTD case WAL_COMPRESSION_ZSTD: - len = ZSTD_compress(dest, COMPRESS_BUFSIZE, source, orig_len, - ZSTD_CLEVEL_DEFAULT); + case WAL_COMPRESSION_ZSTD_1: + case WAL_COMPRESSION_ZSTD_FAST_10: + case WAL_COMPRESSION_ZSTD_FAST_20: + case WAL_COMPRESSION_ZSTD_FAST_40: + { + int level = compression == WAL_COMPRESSION_ZSTD_1 ? 1 : + compression == WAL_COMPRESSION_ZSTD_FAST_10 ? -10 : + compression == WAL_COMPRESSION_ZSTD_FAST_20 ? -20 : + compression == WAL_COMPRESSION_ZSTD_FAST_40 ? -40 : + ZSTD_CLEVEL_DEFAULT; + + len = ZSTD_compress(dest, COMPRESS_BUFSIZE, source, orig_len, level); if (ZSTD_isError(len)) len = -1; break; + } #endif default: diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0f9d522087..0de61d3073 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -1554,6 +1554,10 @@ struct walcompression walmethods[] = { {"zlib", WAL_COMPRESSION_ZLIB}, {"lz4", WAL_COMPRESSION_LZ4}, {"zstd", WAL_COMPRESSION_ZSTD}, + {"zstd-1", WAL_COMPRESSION_ZSTD}, + {"zstd-fast-10",WAL_COMPRESSION_ZSTD}, + {"zstd-fast-20",WAL_COMPRESSION_ZSTD}, + {"zstd-fast-40",WAL_COMPRESSION_ZSTD}, }; /* @@ -1628,6 +1632,14 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) #ifdef USE_ZSTD case WAL_COMPRESSION_ZSTD: + /* + * There aren't actually written into the header - decompression is the + * same. + * case WAL_COMPRESSION_ZSTD_1: + * case WAL_COMPRESSION_ZSTD_FAST_10: + * case WAL_COMPRESSION_ZSTD_FAST_20: + * case WAL_COMPRESSION_ZSTD_FAST_40: + */ decomp_result = ZSTD_decompress(tmp.data, BLCKSZ-bkpb->hole_length, ptr, bkpb->bimg_len); // XXX: ZSTD_getErrorName diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 8031e027aa..667fc4c0c1 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -4728,7 +4728,7 @@ static struct config_enum ConfigureNamesEnum[] = NULL }, &wal_compression_method, - WAL_COMPRESSION_ZSTD, wal_compression_options, + WAL_COMPRESSION_ZSTD_FAST_10, wal_compression_options, NULL, NULL, NULL }, diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index fa8146645d..a435b1a654 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -338,6 +338,10 @@ typedef enum WalCompression WAL_COMPRESSION_ZLIB, WAL_COMPRESSION_LZ4, WAL_COMPRESSION_ZSTD, + WAL_COMPRESSION_ZSTD_1, + WAL_COMPRESSION_ZSTD_FAST_10, /* level = -10 */ + WAL_COMPRESSION_ZSTD_FAST_20, /* level = -20 */ + WAL_COMPRESSION_ZSTD_FAST_40, /* level = -40 */ } WalCompression; extern const char *wal_compression_name(WalCompression compression); -- 2.17.0 --jozmn01XJZjDjM3N-- ^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: Restrict publishing of partitioned table with a foreign table as partition @ 2025-03-24 09:33 Shlok Kyal <[email protected]> 2025-03-24 15:47 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> 0 siblings, 1 reply; 19+ messages in thread From: Shlok Kyal @ 2025-03-24 09:33 UTC (permalink / raw) To: vignesh C <[email protected]>; +Cc: Álvaro Herrera <[email protected]>; Sergey Tatarintsev <[email protected]>; [email protected] On Thu, 20 Feb 2025 at 21:18, vignesh C <[email protected]> wrote: > > On Tue, 18 Feb 2025 at 15:59, Shlok Kyal <[email protected]> wrote: > > > > On Mon, 17 Feb 2025 at 20:13, vignesh C <[email protected]> wrote: > > > > > > On Fri, 14 Feb 2025 at 12:59, Shlok Kyal <[email protected]> wrote: > > > > > > > > I have used the changes suggested by you. Also I have updated the > > > > comments and the function name. > > > > > > There is another concurrency issue possible: > > > +/* Check if a partitioned table has a foreign partition */ > > > +bool > > > +check_partrel_has_foreign_table(Form_pg_class relform) > > > +{ > > > + bool has_foreign_tbl = false; > > > + > > > + if (relform->relkind == RELKIND_PARTITIONED_TABLE) > > > + { > > > + List *relids = NIL; > > > + > > > + relids = find_all_inheritors(relform->oid, NoLock, NULL); > > > > > > Create a publication with publish_via_partition_root as true, hold the > > > execution after check_partrel_has_foreign_table execution finishes. > > > Then parallely execute the following: > > > CREATE TABLE t1(id int) PARTITION BY RANGE(id); > > > CREATE TABLE part1 PARTITION OF t1 FOR VALUES FROM (0) TO (5); > > > CREATE TABLE part2 PARTITION OF t1 FOR VALUES FROM (5) TO (15) > > > PARTITION BY RANGE(id); > > > CREATE FOREIGN TABLE part2_1 PARTITION OF part2 FOR VALUES FROM (10) > > > TO (15) SERVER fdw > > > > > > Now both the partitioned table having foreign table and a publication > > > will be created. > > > > > > > Hi Vignesh, > > > > I have addressed the above issue. If we take a ShareLock on the > > pg_class, we won't be able to create table concurrently, which may > > address the issue. Thoughts? > > I have attached the v8 patch here. > > Since pg_class is a very common table it is not a good idea to take > ShareLock on it. Will it be possible to use pg_partitioned_table table > instead? Hi Vignesh, I have addressed comments and I have modified the patch to take a ShareLock on pg_partitioned_table table. Please find the updated v9 patch. Thanks and Regards, Shlok Kyal Attachments: [application/octet-stream] v9-0001-Restrict-publishing-of-partitioned-table-with-for.patch (26.1K, ../../CANhcyEXbKwQ4ySy0fSzX46N7sauJsvteWMTpYMci_Q03R_ewQA@mail.gmail.com/2-v9-0001-Restrict-publishing-of-partitioned-table-with-for.patch) download | inline diff: From bda55bfffff9f0487730f5ddda34bc0af635faf0 Mon Sep 17 00:00:00 2001 From: Shlok Kyal <[email protected]> Date: Tue, 11 Feb 2025 15:32:55 +0530 Subject: [PATCH v9] Restrict publishing of partitioned table with foreign partition Logical replication of foreign table is not supported and we throw an error in this case. But when create a publication on a partitioned table that has a foreign partition, the initial sync of such table is successful and we should avoid such cases. Current Behaviour in HEAD, when publication is created: 1. with publish_via_partition_root = true Root table is published and the initial data of foreign partitions are replicated. 2. with publish_via_partition_root = false and FOR ALL TABLES All leaf tables except foreign partitions are published. 3. with publish_via_partition_root = false and FOR TABLE/ FOR TABLES IN SCHEMA All leaf tables are published. Initial data of foreign partitions are replicated. With this patch we have following behaviour: 1. with publish_via_partition_root = true We throw an error when we try to publish a foreign partititon. Error is thrown when we try to create a publication on (or add to existing publication) a partitioned table with foreign partition, when try to create a foreign partition and when we try to attach foreign table (or a table with foreign partition) to existing published tables. 2. with publish_via_partition_root = false We skip publishing foreign partition which are part of the publication. This is done by avoid adding foreign partitions in pg_subscription_rel catalog table. We have introduced two functions 'check_partrel_has_foreign_table' and 'check_foreign_tables'. In 'check_partrel_has_foreign_table' we go through the child nodes of a partition and check if it has a foreign table. While doing so, we take a AccessShareLock on each partition table to avoid creation of foreign partition concurrently. In 'check_foreign_tables' if schema id is provided we check for each partitioned table in that schema if it has a foreign partition, or if schema id is not provided we check for each partitioned table in the database if it has a foreign partition. While doing so we take a ShareLock on pg_partitioned_table so no partition table is created concurrently after this check. --- src/backend/catalog/pg_publication.c | 141 +++++++++++++++++++--- src/backend/commands/foreigncmds.c | 47 ++++++++ src/backend/commands/publicationcmds.c | 47 ++++++++ src/backend/commands/tablecmds.c | 53 ++++++++ src/include/catalog/pg_publication.h | 5 + src/test/regress/expected/publication.out | 89 ++++++++++++++ src/test/regress/sql/publication.sql | 81 +++++++++++++ 7 files changed, 446 insertions(+), 17 deletions(-) diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c index d6f94db5d99..d3d8173509e 100644 --- a/src/backend/catalog/pg_publication.c +++ b/src/backend/catalog/pg_publication.c @@ -26,6 +26,7 @@ #include "catalog/partition.h" #include "catalog/pg_inherits.h" #include "catalog/pg_namespace.h" +#include "catalog/pg_partitioned_table.h" #include "catalog/pg_publication.h" #include "catalog/pg_publication_namespace.h" #include "catalog/pg_publication_rel.h" @@ -53,7 +54,7 @@ typedef struct * error if not. */ static void -check_publication_add_relation(Relation targetrel) +check_publication_add_relation(Relation targetrel, Publication *pub) { /* Must be a regular or partitioned table */ if (RelationGetForm(targetrel)->relkind != RELKIND_RELATION && @@ -64,6 +65,19 @@ check_publication_add_relation(Relation targetrel) RelationGetRelationName(targetrel)), errdetail_relkind_not_supported(RelationGetForm(targetrel)->relkind))); + /* + * publish_via_root_partition cannot be true if it is a partitioned table + * and has any foreign partition + */ + if (pub->pubviaroot && + check_partrel_has_foreign_table(RelationGetForm(targetrel))) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot set parameter \"%s\" to true for publication \"%s\"", + "publish_via_partition_root", pub->name), + errdetail("partition table \"%s\" in publication contains a foreign partition", + RelationGetRelationName(targetrel)))); + /* Can't be system table */ if (IsCatalogRelation(targetrel)) ereport(ERROR, @@ -304,7 +318,7 @@ check_and_fetch_column_list(Publication *pub, Oid relid, MemoryContext mcxt, /* * Gets the relations based on the publication partition option for a specified - * relation. + * relation. Foreign tables are not included. */ List * GetPubPartitionOptionRelations(List *result, PublicationPartOpt pub_partopt, @@ -313,25 +327,21 @@ GetPubPartitionOptionRelations(List *result, PublicationPartOpt pub_partopt, if (get_rel_relkind(relid) == RELKIND_PARTITIONED_TABLE && pub_partopt != PUBLICATION_PART_ROOT) { - List *all_parts = find_all_inheritors(relid, NoLock, - NULL); + List *all_parts = find_all_inheritors(relid, NoLock, NULL); - if (pub_partopt == PUBLICATION_PART_ALL) - result = list_concat(result, all_parts); - else if (pub_partopt == PUBLICATION_PART_LEAF) + foreach_oid(partOid, all_parts) { - ListCell *lc; + char relkind = get_rel_relkind(partOid); - foreach(lc, all_parts) - { - Oid partOid = lfirst_oid(lc); + if (relkind == RELKIND_FOREIGN_TABLE) + continue; - if (get_rel_relkind(partOid) != RELKIND_PARTITIONED_TABLE) - result = lappend_oid(result, partOid); - } + if (pub_partopt == PUBLICATION_PART_LEAF && + relkind == RELKIND_PARTITIONED_TABLE) + continue; + + result = lappend_oid(result, partOid); } - else - Assert(false); } else result = lappend_oid(result, relid); @@ -463,7 +473,7 @@ publication_add_relation(Oid pubid, PublicationRelInfo *pri, RelationGetRelationName(targetrel), pub->name))); } - check_publication_add_relation(targetrel); + check_publication_add_relation(targetrel, pub); /* Validate and translate column names into a Bitmapset of attnums. */ attnums = pub_collist_validate(pri->relation, pri->columns); @@ -703,6 +713,13 @@ publication_add_schema(Oid pubid, Oid schemaid, bool if_not_exists) check_publication_add_schema(schemaid); + /* + * If publish_via_partition_root is true, check if schema has any foreign + * partition + */ + if (pub->pubviaroot) + check_foreign_tables(schemaid, pub->name); + /* Form a tuple */ memset(values, 0, sizeof(values)); memset(nulls, false, sizeof(nulls)); @@ -1332,3 +1349,93 @@ pg_get_publication_tables(PG_FUNCTION_ARGS) SRF_RETURN_DONE(funcctx); } + +/* Check if a partitioned table has a foreign partition */ +bool +check_partrel_has_foreign_table(Form_pg_class relform) +{ + bool has_foreign_tbl = false; + + if (relform->relkind == RELKIND_PARTITIONED_TABLE) + { + List *relids = find_all_inheritors(relform->oid, NoLock, NULL); + + foreach_oid(relid, relids) + { + Relation rel = table_open(relid, AccessShareLock); + + if (RelationGetForm(rel)->relkind == RELKIND_FOREIGN_TABLE) + has_foreign_tbl = true; + + table_close(rel, NoLock); + + if (has_foreign_tbl) + break; + } + } + + return has_foreign_tbl; +} + +/* + * If valid schemaid provided, check if the schema has a partition table which + * has a foreign partition. The partition tables in a schema can have partitions + * in other schema. We also need to check if such partitions are foreign + * partition. + * + * If valid schemaid is not provided, we get all partition tables and check if + * it has any foreign partition. We take a lock on partition tables so no new + * foreign partitions are added concurrently. + * + * We take a ShareLock on pg_class to restrict addition of new partitioned table + * which may contain a foreign partition while publication is being created. + */ +void +check_foreign_tables(Oid schemaid, char *pubname) +{ + Relation classRel; + Relation partRel; + ScanKeyData key[3]; + int keycount = 0; + TableScanDesc scan; + HeapTuple tuple; + + classRel = table_open(RelationRelationId, AccessShareLock); + partRel = table_open(PartitionedRelationId, ShareLock); + + /* Get the root nodes of partitioned table */ + ScanKeyInit(&key[keycount++], + Anum_pg_class_relkind, + BTEqualStrategyNumber, F_CHAREQ, + CharGetDatum(RELKIND_PARTITIONED_TABLE)); + + ScanKeyInit(&key[keycount++], + Anum_pg_class_relispartition, + BTEqualStrategyNumber, F_BOOLEQ, + BoolGetDatum(false)); + + /* If schema id is provided check partitioned table in that schema */ + if (OidIsValid(schemaid)) + ScanKeyInit(&key[keycount++], + Anum_pg_class_relnamespace, + BTEqualStrategyNumber, F_OIDEQ, + schemaid); + + scan = table_beginscan_catalog(classRel, keycount, key); + while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) + { + Form_pg_class relForm = (Form_pg_class) GETSTRUCT(tuple); + + if (check_partrel_has_foreign_table(relForm)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot set parameter \"%s\" to true for publication \"%s\"", + "publish_via_partition_root", pubname), + errdetail("partition table \"%s\" in publication contains a foreign partition", + get_rel_name(relForm->oid)))); + } + + table_endscan(scan); + table_close(classRel, AccessShareLock); + table_close(partRel, NoLock); +} diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c index c14e038d54f..f41a24aea39 100644 --- a/src/backend/commands/foreigncmds.c +++ b/src/backend/commands/foreigncmds.c @@ -21,6 +21,7 @@ #include "catalog/dependency.h" #include "catalog/indexing.h" #include "catalog/objectaccess.h" +#include "catalog/partition.h" #include "catalog/pg_foreign_data_wrapper.h" #include "catalog/pg_foreign_server.h" #include "catalog/pg_foreign_table.h" @@ -1423,6 +1424,52 @@ CreateForeignTable(CreateForeignTableStmt *stmt, Oid relid) ftrel = table_open(ForeignTableRelationId, RowExclusiveLock); + /* + * Check if it is a foreign partition and the partitioned table is not + * published or published with publish_via_partition_root option as false. + */ + if (stmt->base.partbound != NULL) + { + RangeVar *root = castNode(RangeVar, lfirst(list_head(stmt->base.inhRelations))); + Relation rootrel = table_openrv(root, AccessShareLock); + + if (RelationGetForm(rootrel)->relkind == RELKIND_PARTITIONED_TABLE) + { + Oid schemaid = RelationGetNamespace(rootrel); + List *puboids = GetRelationPublications(rootrel->rd_id); + List *ancestors; + + puboids = list_concat_unique_oid(puboids, GetAllTablesPublications()); + puboids = list_concat_unique_oid(puboids, GetSchemaPublications(schemaid)); + ancestors = get_partition_ancestors(rootrel->rd_id); + + foreach_oid(ancestor, ancestors) + { + puboids = list_concat_unique_oid(puboids, + GetRelationPublications(ancestor)); + schemaid = get_rel_namespace(ancestor); + puboids = list_concat_unique_oid(puboids, + GetSchemaPublications(schemaid)); + } + list_free(ancestors); + + foreach_oid(puboid, puboids) + { + Publication *pub = GetPublication(puboid); + + if (pub->pubviaroot) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot create table foreign partition \"%s\"", + get_rel_name(relid)), + errdetail("partition table \"%s\" is published with option publish_via_partition_root", + RelationGetRelationName(rootrel)))); + } + } + + table_close(rootrel, AccessShareLock); + } + /* * For now the owner cannot be specified on create. Use effective user ID. */ diff --git a/src/backend/commands/publicationcmds.c b/src/backend/commands/publicationcmds.c index 0b23d94c38e..f74cbb2abb3 100644 --- a/src/backend/commands/publicationcmds.c +++ b/src/backend/commands/publicationcmds.c @@ -915,6 +915,10 @@ CreatePublication(ParseState *pstate, CreatePublicationStmt *stmt) /* Associate objects with the publication. */ if (stmt->for_all_tables) { + /* Check if any partitioned table has foreign partition */ + if (publish_via_partition_root) + check_foreign_tables(InvalidOid, stmt->pubname); + /* Invalidate relcache so that publication info is rebuilt. */ CacheInvalidateRelcacheAll(); } @@ -1080,6 +1084,49 @@ AlterPublicationOptions(ParseState *pstate, AlterPublicationStmt *stmt, } } + /* + * If publish_via_partition_root is set to true, check if the publication + * already have any foreign partition + */ + if (publish_via_partition_root_given && publish_via_partition_root) + { + List *schemaoids = NIL; + List *relids = NIL; + + char *pubname = stmt->pubname; + + if (pubform->puballtables) + check_foreign_tables(InvalidOid, pubname); + + schemaoids = GetPublicationSchemas(pubform->oid); + + foreach_oid(schemaoid, schemaoids) + check_foreign_tables(schemaoid, pubname); + + relids = GetPublicationRelations(pubform->oid, PUBLICATION_PART_ROOT); + + foreach_oid(relid, relids) + { + HeapTuple reltup; + Form_pg_class relform; + + reltup = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); + relform = (Form_pg_class) GETSTRUCT(reltup); + + ReleaseSysCache(reltup); + + if (check_partrel_has_foreign_table(relform)) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot set parameter \"%s\" to true for publication \"%s\"", + "publish_via_partition_root", pubname), + errdetail("partition table \"%s\" in publication contains a foreign partition", + get_rel_name(relid)))); + } + } + } + /* Everything ok, form a new tuple. */ memset(values, 0, sizeof(values)); memset(nulls, false, sizeof(nulls)); diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 1202544ebd0..dac1db2ce50 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -19644,6 +19644,59 @@ ATExecAttachPartition(List **wqueue, Relation rel, PartitionCmd *cmd, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot attach temporary relation of another session as partition"))); + /* + * Check if attachrel is a foreign table or a partitioned table with + * foreign partition and rel is not part of publication with option + * publish_via_partition_root as true. + */ + if (attachrel->rd_rel->relkind == RELKIND_FOREIGN_TABLE || + check_partrel_has_foreign_table(RelationGetForm(attachrel))) + { + Oid schemaid = RelationGetNamespace(rel); + List *puboids = GetRelationPublications(rel->rd_id); + List *ancestors; + char *relname = get_rel_name(rel->rd_id); + char *attachrelname = get_rel_name(attachrel->rd_id); + + puboids = list_concat_unique_oid(puboids, GetAllTablesPublications()); + puboids = list_concat_unique_oid(puboids, GetSchemaPublications(schemaid)); + ancestors = get_partition_ancestors(rel->rd_id); + + foreach_oid(ancestor, ancestors) + { + puboids = list_concat_unique_oid(puboids, + GetRelationPublications(ancestor)); + schemaid = get_rel_namespace(ancestor); + puboids = list_concat_unique_oid(puboids, + GetSchemaPublications(schemaid)); + } + + list_free(ancestors); + + foreach_oid(puboid, puboids) + { + Publication *pub = GetPublication(puboid); + + if (pub->pubviaroot) + { + if (attachrel->rd_rel->relkind == RELKIND_FOREIGN_TABLE) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot attach foreign table \"%s\" to partition table \"%s\"", + attachrelname, relname), + errdetail("partition table \"%s\" is published with option publish_via_partition_root", + relname))); + else + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot attach table \"%s\" with foreign partition to partition table \"%s\"", + attachrelname, relname), + errdetail("partition table \"%s\" is published with option publish_via_partition_root", + relname))); + } + } + } + /* * Check if attachrel has any identity columns or any columns that aren't * in the parent. diff --git a/src/include/catalog/pg_publication.h b/src/include/catalog/pg_publication.h index 48c7d1a8615..50705d12c98 100644 --- a/src/include/catalog/pg_publication.h +++ b/src/include/catalog/pg_publication.h @@ -19,6 +19,7 @@ #include "catalog/genbki.h" #include "catalog/objectaddress.h" +#include "catalog/pg_class.h" #include "catalog/pg_publication_d.h" /* IWYU pragma: export */ /* ---------------- @@ -191,4 +192,8 @@ extern Bitmapset *pub_collist_to_bitmapset(Bitmapset *columns, Datum pubcols, extern Bitmapset *pub_form_cols_map(Relation relation, PublishGencolsType include_gencols_type); +extern bool check_partrel_has_foreign_table(Form_pg_class relform); + +extern void check_foreign_tables(Oid schemaid, char *pubname); + #endif /* PG_PUBLICATION_H */ diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out index 4de96c04f9d..6a0c245bd60 100644 --- a/src/test/regress/expected/publication.out +++ b/src/test/regress/expected/publication.out @@ -1924,6 +1924,95 @@ DROP PUBLICATION pub1; DROP PUBLICATION pub2; DROP TABLE gencols; RESET client_min_messages; +-- ====================================================== +-- Test when foreign table is a partition of a partitioned table on which +-- publication is created +SET client_min_messages = 'ERROR'; +CREATE FOREIGN DATA WRAPPER test_fdw; +CREATE SERVER fdw_server FOREIGN DATA WRAPPER test_fdw; +CREATE SCHEMA sch3; +CREATE TABLE sch3.tmain(id int) PARTITION BY RANGE(id); +CREATE TABLE sch3.part1 PARTITION OF sch3.tmain FOR VALUES FROM (0) TO (5); +CREATE TABLE sch3.part2(id int) PARTITION BY RANGE(id); +CREATE FOREIGN TABLE sch3.part2_1 PARTITION OF sch3.part2 FOR VALUES FROM (5) TO (10) SERVER fdw_server; +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part2 FOR VALUES FROM (5) TO (10); +-- Can't create publications with publish_via_partition_root = true, if table +-- has a foreign partition +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain WITH (publish_via_partition_root); +ERROR: cannot set parameter "publish_via_partition_root" to true for publication "pub1" +DETAIL: partition table "tmain" in publication contains a foreign partition +CREATE PUBLICATION pub1 FOR TABLES IN SCHEMA sch3 WITH (publish_via_partition_root); +ERROR: cannot set parameter "publish_via_partition_root" to true for publication "pub1" +DETAIL: partition table "tmain" in publication contains a foreign partition +CREATE PUBLICATION pub1 FOR ALL TABLES WITH (publish_via_partition_root); +ERROR: cannot set parameter "publish_via_partition_root" to true for publication "pub1" +DETAIL: partition table "tmain" in publication contains a foreign partition +-- Test when a partitioned table with foreign table as a partition is attached +-- to partitioned table which is already published +ALTER TABLE sch3.tmain DETACH PARTITION sch3.part2; +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain WITH (publish_via_partition_root); +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part2 FOR VALUES FROM (5) TO (10); +ERROR: cannot attach table "part2" with foreign partition to partition table "tmain" +DETAIL: partition table "tmain" is published with option publish_via_partition_root +-- Can't create foreign partition of published table with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch3.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; +ERROR: cannot create table foreign partition "part3_1" +DETAIL: partition table "tmain" is published with option publish_via_partition_root +-- Can't attach foreign partition to published table with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_2(id int) SERVER fdw_server; +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); +ERROR: cannot attach foreign table "part3_2" to partition table "tmain" +DETAIL: partition table "tmain" is published with option publish_via_partition_root +CREATE SCHEMA sch4; +CREATE TABLE sch4.tmain(id int) PARTITION BY RANGE(id); +-- publication created with FOR TABLES IN SCHEMA +DROP PUBLICATION pub1; +CREATE PUBLICATION pub1 FOR TABLES IN SCHEMA sch4 WITH (publish_via_partition_root); +-- Can't create foreign partition of published table with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch4.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; +ERROR: cannot create table foreign partition "part3_1" +DETAIL: partition table "tmain" is published with option publish_via_partition_root +-- Can't attach foreign partition to published table with +-- publish_via_partition_root = true +ALTER TABLE sch4.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); +ERROR: cannot attach foreign table "part3_2" to partition table "tmain" +DETAIL: partition table "tmain" is published with option publish_via_partition_root +DROP PUBLICATION pub1; +-- Test with publish_via_partition_root = false +-- Foreign partitions are skipped by default +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain; +CREATE PUBLICATION pub2 FOR TABLES IN SCHEMA sch3; +CREATE PUBLICATION pub3 FOR ALL TABLES; +-- Create foreign partition of published table with +-- publish_via_partition_root = false +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch3.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; +-- Attach foreign partition to published table +-- publish_via_partition_root = false +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); +-- Check the published tables +SELECT pubname, tablename FROM pg_publication_tables WHERE schemaname in ('sch3', 'sch4') ORDER BY pubname, tablename; + pubname | tablename +---------+----------- + pub1 | part1 + pub2 | part1 + pub3 | part1 +(3 rows) + +-- Can't alter publish_via_partition_root to true, if publication already have +-- foreign partition +ALTER PUBLICATION pub1 SET (publish_via_partition_root); +ERROR: cannot set parameter "publish_via_partition_root" to true for publication "pub1" +DETAIL: partition table "tmain" in publication contains a foreign partition +DROP PUBLICATION pub1; +DROP PUBLICATION pub2; +DROP PUBLICATION pub3; +DROP SCHEMA sch3 CASCADE; +DROP SCHEMA sch4 CASCADE; +DROP SERVER fdw_server; +DROP FOREIGN DATA WRAPPER test_fdw; RESET SESSION AUTHORIZATION; DROP ROLE regress_publication_user, regress_publication_user2; DROP ROLE regress_publication_user_dummy; diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql index 68001de4000..49c9d98b668 100644 --- a/src/test/regress/sql/publication.sql +++ b/src/test/regress/sql/publication.sql @@ -1223,6 +1223,87 @@ DROP PUBLICATION pub2; DROP TABLE gencols; RESET client_min_messages; +-- ====================================================== + +-- Test when foreign table is a partition of a partitioned table on which +-- publication is created +SET client_min_messages = 'ERROR'; +CREATE FOREIGN DATA WRAPPER test_fdw; +CREATE SERVER fdw_server FOREIGN DATA WRAPPER test_fdw; + +CREATE SCHEMA sch3; +CREATE TABLE sch3.tmain(id int) PARTITION BY RANGE(id); +CREATE TABLE sch3.part1 PARTITION OF sch3.tmain FOR VALUES FROM (0) TO (5); +CREATE TABLE sch3.part2(id int) PARTITION BY RANGE(id); +CREATE FOREIGN TABLE sch3.part2_1 PARTITION OF sch3.part2 FOR VALUES FROM (5) TO (10) SERVER fdw_server; +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part2 FOR VALUES FROM (5) TO (10); + +-- Can't create publications with publish_via_partition_root = true, if table +-- has a foreign partition +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain WITH (publish_via_partition_root); +CREATE PUBLICATION pub1 FOR TABLES IN SCHEMA sch3 WITH (publish_via_partition_root); +CREATE PUBLICATION pub1 FOR ALL TABLES WITH (publish_via_partition_root); + +-- Test when a partitioned table with foreign table as a partition is attached +-- to partitioned table which is already published +ALTER TABLE sch3.tmain DETACH PARTITION sch3.part2; +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain WITH (publish_via_partition_root); +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part2 FOR VALUES FROM (5) TO (10); + +-- Can't create foreign partition of published table with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch3.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; + +-- Can't attach foreign partition to published table with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_2(id int) SERVER fdw_server; +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); + +CREATE SCHEMA sch4; +CREATE TABLE sch4.tmain(id int) PARTITION BY RANGE(id); + +-- publication created with FOR TABLES IN SCHEMA +DROP PUBLICATION pub1; +CREATE PUBLICATION pub1 FOR TABLES IN SCHEMA sch4 WITH (publish_via_partition_root); + +-- Can't create foreign partition of published table with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch4.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; + +-- Can't attach foreign partition to published table with +-- publish_via_partition_root = true +ALTER TABLE sch4.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); +DROP PUBLICATION pub1; + +-- Test with publish_via_partition_root = false +-- Foreign partitions are skipped by default +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain; +CREATE PUBLICATION pub2 FOR TABLES IN SCHEMA sch3; +CREATE PUBLICATION pub3 FOR ALL TABLES; + +-- Create foreign partition of published table with +-- publish_via_partition_root = false +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch3.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; + +-- Attach foreign partition to published table +-- publish_via_partition_root = false +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); + +-- Check the published tables +SELECT pubname, tablename FROM pg_publication_tables WHERE schemaname in ('sch3', 'sch4') ORDER BY pubname, tablename; + +-- Can't alter publish_via_partition_root to true, if publication already have +-- foreign partition +ALTER PUBLICATION pub1 SET (publish_via_partition_root); + +DROP PUBLICATION pub1; +DROP PUBLICATION pub2; +DROP PUBLICATION pub3; +DROP SCHEMA sch3 CASCADE; +DROP SCHEMA sch4 CASCADE; +DROP SERVER fdw_server; +DROP FOREIGN DATA WRAPPER test_fdw; + RESET SESSION AUTHORIZATION; DROP ROLE regress_publication_user, regress_publication_user2; DROP ROLE regress_publication_user_dummy; -- 2.34.1 ^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: Restrict publishing of partitioned table with a foreign table as partition 2025-03-24 09:33 Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> @ 2025-03-24 15:47 ` Álvaro Herrera <[email protected]> 2025-03-28 09:43 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 0 siblings, 1 reply; 19+ messages in thread From: Álvaro Herrera @ 2025-03-24 15:47 UTC (permalink / raw) To: Shlok Kyal <[email protected]>; +Cc: vignesh C <[email protected]>; Sergey Tatarintsev <[email protected]>; [email protected] One thing that bothers me a bit about this is that there's no single code comment where this restriction it documented in full; in fact it doesn't seem documented anywhere, only in the commit message. I think check_foreign_tables() is a good place to add an explanatory comment; other places can reference that. For instance, add something like /* * Protect against including foreign tables that are partitions of * partitioned tables published by the given publication. This would * not work properly, because <!-- explain reason -->, so we disallow * the case here and in all DDL commands that would end up creating * such a case indirectly. */ Then for instance in check_publication_add_relation() and ATExecAttachPartition() you comment would say /* if the would-be partition is a foreign table, verify that the partitioned table is not in a publication with publish_via_root=false. See check_foreign_tables for details */ Also, surely we should document this restriction in the SGML docs somewhere. Would it be better if check_partrel_has_foreign_table() used RelationGetPartitionDesc(omit_detached=true) instead of find_all_inheritors()? I'm wary of all those accesses of subscription/publication catalogs in DDL code. Maybe I worry about nothing, but I cannot but feel that we're missing one layer of abstraction there (including possibly some caching on top of syscache). I think this castNode(RangeVar, lfirst(list_head(stmt->base.inhRelations))); is better written linitial_node(RangeVar, stmt->base.inhRelations); -- Álvaro Herrera PostgreSQL Developer — https://www.EnterpriseDB.com/ "Tiene valor aquel que admite que es un cobarde" (Fernandel) ^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: Restrict publishing of partitioned table with a foreign table as partition 2025-03-24 09:33 Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-03-24 15:47 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> @ 2025-03-28 09:43 ` Shlok Kyal <[email protected]> 2025-03-28 11:04 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> 0 siblings, 1 reply; 19+ messages in thread From: Shlok Kyal @ 2025-03-28 09:43 UTC (permalink / raw) To: Álvaro Herrera <[email protected]>; +Cc: vignesh C <[email protected]>; Sergey Tatarintsev <[email protected]>; [email protected] On Mon, 24 Mar 2025 at 21:17, Álvaro Herrera <[email protected]> wrote: > > One thing that bothers me a bit about this is that there's no single > code comment where this restriction it documented in full; in fact it > doesn't seem documented anywhere, only in the commit message. > > I think check_foreign_tables() is a good place to add an explanatory > comment; other places can reference that. For instance, add something > like > > /* > * Protect against including foreign tables that are partitions of > * partitioned tables published by the given publication. This would > * not work properly, because <!-- explain reason -->, so we disallow > * the case here and in all DDL commands that would end up creating > * such a case indirectly. > */ > > Then for instance in check_publication_add_relation() and > ATExecAttachPartition() you comment would say /* if the would-be > partition is a foreign table, verify that the partitioned table is not > in a publication with publish_via_root=false. See check_foreign_tables > for details */ > I have added comments as suggested above. > Also, surely we should document this restriction in the SGML docs > somewhere. > I have added comment in create_publication.sgml > > Would it be better if check_partrel_has_foreign_table() used > RelationGetPartitionDesc(omit_detached=true) instead of > find_all_inheritors()? > > I'm wary of all those accesses of subscription/publication catalogs in > DDL code. Maybe I worry about nothing, but I cannot but feel that we're > missing one layer of abstraction there (including possibly some caching > on top of syscache). > I also think using RelationGetPartitionDesc would be better and made the change. > I think this > castNode(RangeVar, lfirst(list_head(stmt->base.inhRelations))); > is better written > linitial_node(RangeVar, stmt->base.inhRelations); Fixed. I have attached the updated v10 patch with the above changes. Thanks and Regards, Shlok Kyal Attachments: [application/octet-stream] v10-0001-Restrict-publishing-of-partitioned-table-with-fo.patch (28.3K, ../../CANhcyEU5R4qdkO8SLSswRWgBRDPEaapAQ61rCjjDBg4RNmmsrw@mail.gmail.com/2-v10-0001-Restrict-publishing-of-partitioned-table-with-fo.patch) download | inline diff: From f8b13f85faae6b8bd4594ac84504ac8fa0cd77d0 Mon Sep 17 00:00:00 2001 From: Shlok Kyal <[email protected]> Date: Fri, 28 Mar 2025 11:15:09 +0530 Subject: [PATCH v10] Restrict publishing of partitioned table with foreign partition Logical replication of foreign table is not supported and we throw an error in this case. But when create a publication on a partitioned table that has a foreign partition, the initial sync of such table is successful and we should avoid such cases. Current Behaviour in HEAD, when publication is created: 1. with publish_via_partition_root = true Root table is published and the initial data of foreign partitions are replicated. 2. with publish_via_partition_root = false and FOR ALL TABLES All leaf tables except foreign partitions are published. 3. with publish_via_partition_root = false and FOR TABLE/ FOR TABLES IN SCHEMA All leaf tables are published. Initial data of foreign partitions are replicated. With this patch we have following behaviour: 1. with publish_via_partition_root = true We throw an error when we try to publish a foreign partititon. Error is thrown when we try to create a publication on (or add to existing publication) a partitioned table with foreign partition, when try to create a foreign partition and when we try to attach foreign table (or a table with foreign partition) to existing published tables. 2. with publish_via_partition_root = false We skip publishing foreign partition which are part of the publication. This is done by avoid adding foreign partitions in pg_subscription_rel catalog table. We have introduced two functions 'check_partrel_has_foreign_table' and 'check_foreign_tables'. In 'check_partrel_has_foreign_table' we go through the child nodes of a partition and check if it has a foreign table. While doing so, we take a AccessShareLock on each partition table to avoid creation of foreign partition concurrently. In 'check_foreign_tables' if schema id is provided we check for each partitioned table in that schema if it has a foreign partition, or if schema id is not provided we check for each partitioned table in the database if it has a foreign partition. While doing so we take a ShareLock on pg_partitioned_table so no partition table is created concurrently after this check. --- doc/src/sgml/ref/create_publication.sgml | 5 + src/backend/catalog/pg_publication.c | 154 +++++++++++++++++++--- src/backend/commands/foreigncmds.c | 48 +++++++ src/backend/commands/publicationcmds.c | 43 ++++++ src/backend/commands/tablecmds.c | 57 ++++++++ src/include/catalog/pg_publication.h | 5 + src/test/regress/expected/publication.out | 89 +++++++++++++ src/test/regress/sql/publication.sql | 81 ++++++++++++ 8 files changed, 465 insertions(+), 17 deletions(-) diff --git a/doc/src/sgml/ref/create_publication.sgml b/doc/src/sgml/ref/create_publication.sgml index 73f0c8d89fb..c4e993b1c1a 100644 --- a/doc/src/sgml/ref/create_publication.sgml +++ b/doc/src/sgml/ref/create_publication.sgml @@ -257,6 +257,11 @@ CREATE PUBLICATION <replaceable class="parameter">name</replaceable> If this is enabled, <literal>TRUNCATE</literal> operations performed directly on partitions are not replicated. </para> + + <para> + If this is enabled, we cannot include a foreign table or a partitioned + table with a foreign partition in the publication. + </para> </listitem> </varlistentry> </variablelist></para> diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c index d6f94db5d99..0daa5be3469 100644 --- a/src/backend/catalog/pg_publication.c +++ b/src/backend/catalog/pg_publication.c @@ -26,12 +26,14 @@ #include "catalog/partition.h" #include "catalog/pg_inherits.h" #include "catalog/pg_namespace.h" +#include "catalog/pg_partitioned_table.h" #include "catalog/pg_publication.h" #include "catalog/pg_publication_namespace.h" #include "catalog/pg_publication_rel.h" #include "catalog/pg_type.h" #include "commands/publicationcmds.h" #include "funcapi.h" +#include "partitioning/partdesc.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/catcache.h" @@ -53,7 +55,7 @@ typedef struct * error if not. */ static void -check_publication_add_relation(Relation targetrel) +check_publication_add_relation(Relation targetrel, Publication *pub) { /* Must be a regular or partitioned table */ if (RelationGetForm(targetrel)->relkind != RELKIND_RELATION && @@ -64,6 +66,18 @@ check_publication_add_relation(Relation targetrel) RelationGetRelationName(targetrel)), errdetail_relkind_not_supported(RelationGetForm(targetrel)->relkind))); + /* + * publish_via_root_partition cannot be true if it is a partitioned table + * and has any foreign partition. See check_foreign_tables for details. + */ + if (pub->pubviaroot && check_partrel_has_foreign_table(targetrel)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot set parameter \"%s\" to true for publication \"%s\"", + "publish_via_partition_root", pub->name), + errdetail("partition table \"%s\" in publication contains a foreign partition", + RelationGetRelationName(targetrel)))); + /* Can't be system table */ if (IsCatalogRelation(targetrel)) ereport(ERROR, @@ -304,7 +318,7 @@ check_and_fetch_column_list(Publication *pub, Oid relid, MemoryContext mcxt, /* * Gets the relations based on the publication partition option for a specified - * relation. + * relation. Foreign tables are not included. */ List * GetPubPartitionOptionRelations(List *result, PublicationPartOpt pub_partopt, @@ -313,25 +327,21 @@ GetPubPartitionOptionRelations(List *result, PublicationPartOpt pub_partopt, if (get_rel_relkind(relid) == RELKIND_PARTITIONED_TABLE && pub_partopt != PUBLICATION_PART_ROOT) { - List *all_parts = find_all_inheritors(relid, NoLock, - NULL); + List *all_parts = find_all_inheritors(relid, NoLock, NULL); - if (pub_partopt == PUBLICATION_PART_ALL) - result = list_concat(result, all_parts); - else if (pub_partopt == PUBLICATION_PART_LEAF) + foreach_oid(partOid, all_parts) { - ListCell *lc; + char relkind = get_rel_relkind(partOid); - foreach(lc, all_parts) - { - Oid partOid = lfirst_oid(lc); + if (relkind == RELKIND_FOREIGN_TABLE) + continue; - if (get_rel_relkind(partOid) != RELKIND_PARTITIONED_TABLE) - result = lappend_oid(result, partOid); - } + if (pub_partopt == PUBLICATION_PART_LEAF && + relkind == RELKIND_PARTITIONED_TABLE) + continue; + + result = lappend_oid(result, partOid); } - else - Assert(false); } else result = lappend_oid(result, relid); @@ -463,7 +473,7 @@ publication_add_relation(Oid pubid, PublicationRelInfo *pri, RelationGetRelationName(targetrel), pub->name))); } - check_publication_add_relation(targetrel); + check_publication_add_relation(targetrel, pub); /* Validate and translate column names into a Bitmapset of attnums. */ attnums = pub_collist_validate(pri->relation, pri->columns); @@ -703,6 +713,13 @@ publication_add_schema(Oid pubid, Oid schemaid, bool if_not_exists) check_publication_add_schema(schemaid); + /* + * If publish_via_partition_root is true, check if schema has any foreign + * partition + */ + if (pub->pubviaroot) + check_foreign_tables(schemaid, pub->name); + /* Form a tuple */ memset(values, 0, sizeof(values)); memset(nulls, false, sizeof(nulls)); @@ -1332,3 +1349,106 @@ pg_get_publication_tables(PG_FUNCTION_ARGS) SRF_RETURN_DONE(funcctx); } + +/* Check if a partitioned table has a foreign partition */ +bool +check_partrel_has_foreign_table(Relation rel) +{ + bool has_foreign_tbl = false; + + if (RelationGetForm(rel)->relkind == RELKIND_PARTITIONED_TABLE) + { + PartitionDesc pd = RelationGetPartitionDesc(rel, true); + + for (int i = 0; i < pd->nparts; i++) + { + Relation childrel = table_open(pd->oids[i], AccessShareLock); + + if (RelationGetForm(childrel)->relkind == RELKIND_FOREIGN_TABLE) + has_foreign_tbl = true; + else + has_foreign_tbl = check_partrel_has_foreign_table(childrel); + + table_close(childrel, NoLock); + + if (has_foreign_tbl) + break; + } + } + + return has_foreign_tbl; +} + +/* + * Protect against including foreign tables that are partitions of partitioned + * tables published by the given publication when publish_via_root_partition is + * true. This will not work correctly as the initial data from the foreign + * table can be replicated by the tablesync worker even though replication of + * foreign table is not supported. So we disallow the case here and in all DDL + * commands that would end up creating such a case indirectly. + * + * If valid schemaid provided, check if the schema has a partition table which + * has a foreign partition. The partition tables in a schema can have partitions + * in other schema. We also need to check if such partitions are foreign + * partition. + * + * If valid schemaid is not provided, we get all partition tables and check if + * it has any foreign partition. We take a lock on partition tables so no new + * foreign partitions are added concurrently. + * + * We take a ShareLock on pg_partitioned_table to restrict addition of new + * partitioned table which may contain a foreign partition while publication is + * being created. + */ +void +check_foreign_tables(Oid schemaid, char *pubname) +{ + Relation classRel; + Relation partRel; + ScanKeyData key[3]; + int keycount = 0; + TableScanDesc scan; + HeapTuple tuple; + + classRel = table_open(RelationRelationId, AccessShareLock); + partRel = table_open(PartitionedRelationId, ShareLock); + + /* Get the root nodes of partitioned table */ + ScanKeyInit(&key[keycount++], + Anum_pg_class_relkind, + BTEqualStrategyNumber, F_CHAREQ, + CharGetDatum(RELKIND_PARTITIONED_TABLE)); + + ScanKeyInit(&key[keycount++], + Anum_pg_class_relispartition, + BTEqualStrategyNumber, F_BOOLEQ, + BoolGetDatum(false)); + + /* If schema id is provided check partitioned table in that schema */ + if (OidIsValid(schemaid)) + ScanKeyInit(&key[keycount++], + Anum_pg_class_relnamespace, + BTEqualStrategyNumber, F_OIDEQ, + schemaid); + + scan = table_beginscan_catalog(classRel, keycount, key); + while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) + { + Form_pg_class relForm = (Form_pg_class) GETSTRUCT(tuple); + Relation pubrel = table_open(relForm->oid, AccessShareLock); + + if (check_partrel_has_foreign_table(pubrel)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot set parameter \"%s\" to true for publication \"%s\"", + "publish_via_partition_root", pubname), + errdetail("partition table \"%s\" in publication contains a foreign partition", + get_rel_name(relForm->oid)))); + + table_close(pubrel, NoLock); + } + + table_endscan(scan); + table_close(classRel, AccessShareLock); + table_close(partRel, NoLock); +} diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c index c14e038d54f..abc23baa59e 100644 --- a/src/backend/commands/foreigncmds.c +++ b/src/backend/commands/foreigncmds.c @@ -21,6 +21,7 @@ #include "catalog/dependency.h" #include "catalog/indexing.h" #include "catalog/objectaccess.h" +#include "catalog/partition.h" #include "catalog/pg_foreign_data_wrapper.h" #include "catalog/pg_foreign_server.h" #include "catalog/pg_foreign_table.h" @@ -1423,6 +1424,53 @@ CreateForeignTable(CreateForeignTableStmt *stmt, Oid relid) ftrel = table_open(ForeignTableRelationId, RowExclusiveLock); + /* + * Check if it is a foreign partition and the partitioned table is not + * published or published with publish_via_partition_root option as false. + * See check_foreign_tables for details. + */ + if (stmt->base.partbound != NULL) + { + RangeVar *root = linitial_node(RangeVar, stmt->base.inhRelations); + Relation rootrel = table_openrv(root, AccessShareLock); + + if (RelationGetForm(rootrel)->relkind == RELKIND_PARTITIONED_TABLE) + { + Oid schemaid = RelationGetNamespace(rootrel); + List *puboids = GetRelationPublications(rootrel->rd_id); + List *ancestors; + + puboids = list_concat_unique_oid(puboids, GetAllTablesPublications()); + puboids = list_concat_unique_oid(puboids, GetSchemaPublications(schemaid)); + ancestors = get_partition_ancestors(rootrel->rd_id); + + foreach_oid(ancestor, ancestors) + { + puboids = list_concat_unique_oid(puboids, + GetRelationPublications(ancestor)); + schemaid = get_rel_namespace(ancestor); + puboids = list_concat_unique_oid(puboids, + GetSchemaPublications(schemaid)); + } + list_free(ancestors); + + foreach_oid(puboid, puboids) + { + Publication *pub = GetPublication(puboid); + + if (pub->pubviaroot) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot create table foreign partition \"%s\"", + get_rel_name(relid)), + errdetail("partition table \"%s\" is published with option publish_via_partition_root", + RelationGetRelationName(rootrel)))); + } + } + + table_close(rootrel, AccessShareLock); + } + /* * For now the owner cannot be specified on create. Use effective user ID. */ diff --git a/src/backend/commands/publicationcmds.c b/src/backend/commands/publicationcmds.c index 0b23d94c38e..e5fa1bf5b09 100644 --- a/src/backend/commands/publicationcmds.c +++ b/src/backend/commands/publicationcmds.c @@ -915,6 +915,10 @@ CreatePublication(ParseState *pstate, CreatePublicationStmt *stmt) /* Associate objects with the publication. */ if (stmt->for_all_tables) { + /* Check if any partitioned table has foreign partition */ + if (publish_via_partition_root) + check_foreign_tables(InvalidOid, stmt->pubname); + /* Invalidate relcache so that publication info is rebuilt. */ CacheInvalidateRelcacheAll(); } @@ -1080,6 +1084,45 @@ AlterPublicationOptions(ParseState *pstate, AlterPublicationStmt *stmt, } } + /* + * If publish_via_partition_root is set to true, check if the publication + * already have any foreign partition. See check_foreign_tables for details. + */ + if (publish_via_partition_root_given && publish_via_partition_root) + { + List *schemaoids = NIL; + List *relids = NIL; + + char *pubname = stmt->pubname; + + if (pubform->puballtables) + check_foreign_tables(InvalidOid, pubname); + + schemaoids = GetPublicationSchemas(pubform->oid); + + foreach_oid(schemaoid, schemaoids) + check_foreign_tables(schemaoid, pubname); + + relids = GetPublicationRelations(pubform->oid, PUBLICATION_PART_ROOT); + + foreach_oid(relid, relids) + { + Relation pubrel = table_open(relid, AccessShareLock); + + if (check_partrel_has_foreign_table(pubrel)) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot set parameter \"%s\" to true for publication \"%s\"", + "publish_via_partition_root", pubname), + errdetail("partition table \"%s\" in publication contains a foreign partition", + get_rel_name(relid)))); + } + + table_close(pubrel, NoLock); + } + } + /* Everything ok, form a new tuple. */ memset(values, 0, sizeof(values)); memset(nulls, false, sizeof(nulls)); diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index afb25007613..fc4db92bde7 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -19570,6 +19570,10 @@ QueuePartitionConstraintValidation(List **wqueue, Relation scanrel, * ALTER TABLE <name> ATTACH PARTITION <partition-name> FOR VALUES * * Return the address of the newly attached partition. + * + * If the would-be partition is a foreign table or a partitioned table with + * foreign partition, verify that the partitioned table is not in a publication + * with publish_via_partition_root=true. See check_foreign_tables for details. */ static ObjectAddress ATExecAttachPartition(List **wqueue, Relation rel, PartitionCmd *cmd, @@ -19714,6 +19718,59 @@ ATExecAttachPartition(List **wqueue, Relation rel, PartitionCmd *cmd, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot attach temporary relation of another session as partition"))); + /* + * Check if attachrel is a foreign table or a partitioned table with + * foreign partition and rel is not part of publication with option + * publish_via_partition_root as true. + */ + if (attachrel->rd_rel->relkind == RELKIND_FOREIGN_TABLE || + check_partrel_has_foreign_table(attachrel)) + { + Oid schemaid = RelationGetNamespace(rel); + List *puboids = GetRelationPublications(rel->rd_id); + List *ancestors; + char *relname = get_rel_name(rel->rd_id); + char *attachrelname = get_rel_name(attachrel->rd_id); + + puboids = list_concat_unique_oid(puboids, GetAllTablesPublications()); + puboids = list_concat_unique_oid(puboids, GetSchemaPublications(schemaid)); + ancestors = get_partition_ancestors(rel->rd_id); + + foreach_oid(ancestor, ancestors) + { + puboids = list_concat_unique_oid(puboids, + GetRelationPublications(ancestor)); + schemaid = get_rel_namespace(ancestor); + puboids = list_concat_unique_oid(puboids, + GetSchemaPublications(schemaid)); + } + + list_free(ancestors); + + foreach_oid(puboid, puboids) + { + Publication *pub = GetPublication(puboid); + + if (pub->pubviaroot) + { + if (attachrel->rd_rel->relkind == RELKIND_FOREIGN_TABLE) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot attach foreign table \"%s\" to partition table \"%s\"", + attachrelname, relname), + errdetail("partition table \"%s\" is published with option publish_via_partition_root", + relname))); + else + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot attach table \"%s\" with foreign partition to partition table \"%s\"", + attachrelname, relname), + errdetail("partition table \"%s\" is published with option publish_via_partition_root", + relname))); + } + } + } + /* * Check if attachrel has any identity columns or any columns that aren't * in the parent. diff --git a/src/include/catalog/pg_publication.h b/src/include/catalog/pg_publication.h index 48c7d1a8615..e2919da2541 100644 --- a/src/include/catalog/pg_publication.h +++ b/src/include/catalog/pg_publication.h @@ -19,6 +19,7 @@ #include "catalog/genbki.h" #include "catalog/objectaddress.h" +#include "catalog/pg_class.h" #include "catalog/pg_publication_d.h" /* IWYU pragma: export */ /* ---------------- @@ -191,4 +192,8 @@ extern Bitmapset *pub_collist_to_bitmapset(Bitmapset *columns, Datum pubcols, extern Bitmapset *pub_form_cols_map(Relation relation, PublishGencolsType include_gencols_type); +extern bool check_partrel_has_foreign_table(Relation rel); + +extern void check_foreign_tables(Oid schemaid, char *pubname); + #endif /* PG_PUBLICATION_H */ diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out index 4de96c04f9d..6a0c245bd60 100644 --- a/src/test/regress/expected/publication.out +++ b/src/test/regress/expected/publication.out @@ -1924,6 +1924,95 @@ DROP PUBLICATION pub1; DROP PUBLICATION pub2; DROP TABLE gencols; RESET client_min_messages; +-- ====================================================== +-- Test when foreign table is a partition of a partitioned table on which +-- publication is created +SET client_min_messages = 'ERROR'; +CREATE FOREIGN DATA WRAPPER test_fdw; +CREATE SERVER fdw_server FOREIGN DATA WRAPPER test_fdw; +CREATE SCHEMA sch3; +CREATE TABLE sch3.tmain(id int) PARTITION BY RANGE(id); +CREATE TABLE sch3.part1 PARTITION OF sch3.tmain FOR VALUES FROM (0) TO (5); +CREATE TABLE sch3.part2(id int) PARTITION BY RANGE(id); +CREATE FOREIGN TABLE sch3.part2_1 PARTITION OF sch3.part2 FOR VALUES FROM (5) TO (10) SERVER fdw_server; +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part2 FOR VALUES FROM (5) TO (10); +-- Can't create publications with publish_via_partition_root = true, if table +-- has a foreign partition +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain WITH (publish_via_partition_root); +ERROR: cannot set parameter "publish_via_partition_root" to true for publication "pub1" +DETAIL: partition table "tmain" in publication contains a foreign partition +CREATE PUBLICATION pub1 FOR TABLES IN SCHEMA sch3 WITH (publish_via_partition_root); +ERROR: cannot set parameter "publish_via_partition_root" to true for publication "pub1" +DETAIL: partition table "tmain" in publication contains a foreign partition +CREATE PUBLICATION pub1 FOR ALL TABLES WITH (publish_via_partition_root); +ERROR: cannot set parameter "publish_via_partition_root" to true for publication "pub1" +DETAIL: partition table "tmain" in publication contains a foreign partition +-- Test when a partitioned table with foreign table as a partition is attached +-- to partitioned table which is already published +ALTER TABLE sch3.tmain DETACH PARTITION sch3.part2; +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain WITH (publish_via_partition_root); +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part2 FOR VALUES FROM (5) TO (10); +ERROR: cannot attach table "part2" with foreign partition to partition table "tmain" +DETAIL: partition table "tmain" is published with option publish_via_partition_root +-- Can't create foreign partition of published table with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch3.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; +ERROR: cannot create table foreign partition "part3_1" +DETAIL: partition table "tmain" is published with option publish_via_partition_root +-- Can't attach foreign partition to published table with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_2(id int) SERVER fdw_server; +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); +ERROR: cannot attach foreign table "part3_2" to partition table "tmain" +DETAIL: partition table "tmain" is published with option publish_via_partition_root +CREATE SCHEMA sch4; +CREATE TABLE sch4.tmain(id int) PARTITION BY RANGE(id); +-- publication created with FOR TABLES IN SCHEMA +DROP PUBLICATION pub1; +CREATE PUBLICATION pub1 FOR TABLES IN SCHEMA sch4 WITH (publish_via_partition_root); +-- Can't create foreign partition of published table with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch4.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; +ERROR: cannot create table foreign partition "part3_1" +DETAIL: partition table "tmain" is published with option publish_via_partition_root +-- Can't attach foreign partition to published table with +-- publish_via_partition_root = true +ALTER TABLE sch4.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); +ERROR: cannot attach foreign table "part3_2" to partition table "tmain" +DETAIL: partition table "tmain" is published with option publish_via_partition_root +DROP PUBLICATION pub1; +-- Test with publish_via_partition_root = false +-- Foreign partitions are skipped by default +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain; +CREATE PUBLICATION pub2 FOR TABLES IN SCHEMA sch3; +CREATE PUBLICATION pub3 FOR ALL TABLES; +-- Create foreign partition of published table with +-- publish_via_partition_root = false +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch3.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; +-- Attach foreign partition to published table +-- publish_via_partition_root = false +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); +-- Check the published tables +SELECT pubname, tablename FROM pg_publication_tables WHERE schemaname in ('sch3', 'sch4') ORDER BY pubname, tablename; + pubname | tablename +---------+----------- + pub1 | part1 + pub2 | part1 + pub3 | part1 +(3 rows) + +-- Can't alter publish_via_partition_root to true, if publication already have +-- foreign partition +ALTER PUBLICATION pub1 SET (publish_via_partition_root); +ERROR: cannot set parameter "publish_via_partition_root" to true for publication "pub1" +DETAIL: partition table "tmain" in publication contains a foreign partition +DROP PUBLICATION pub1; +DROP PUBLICATION pub2; +DROP PUBLICATION pub3; +DROP SCHEMA sch3 CASCADE; +DROP SCHEMA sch4 CASCADE; +DROP SERVER fdw_server; +DROP FOREIGN DATA WRAPPER test_fdw; RESET SESSION AUTHORIZATION; DROP ROLE regress_publication_user, regress_publication_user2; DROP ROLE regress_publication_user_dummy; diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql index 68001de4000..49c9d98b668 100644 --- a/src/test/regress/sql/publication.sql +++ b/src/test/regress/sql/publication.sql @@ -1223,6 +1223,87 @@ DROP PUBLICATION pub2; DROP TABLE gencols; RESET client_min_messages; +-- ====================================================== + +-- Test when foreign table is a partition of a partitioned table on which +-- publication is created +SET client_min_messages = 'ERROR'; +CREATE FOREIGN DATA WRAPPER test_fdw; +CREATE SERVER fdw_server FOREIGN DATA WRAPPER test_fdw; + +CREATE SCHEMA sch3; +CREATE TABLE sch3.tmain(id int) PARTITION BY RANGE(id); +CREATE TABLE sch3.part1 PARTITION OF sch3.tmain FOR VALUES FROM (0) TO (5); +CREATE TABLE sch3.part2(id int) PARTITION BY RANGE(id); +CREATE FOREIGN TABLE sch3.part2_1 PARTITION OF sch3.part2 FOR VALUES FROM (5) TO (10) SERVER fdw_server; +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part2 FOR VALUES FROM (5) TO (10); + +-- Can't create publications with publish_via_partition_root = true, if table +-- has a foreign partition +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain WITH (publish_via_partition_root); +CREATE PUBLICATION pub1 FOR TABLES IN SCHEMA sch3 WITH (publish_via_partition_root); +CREATE PUBLICATION pub1 FOR ALL TABLES WITH (publish_via_partition_root); + +-- Test when a partitioned table with foreign table as a partition is attached +-- to partitioned table which is already published +ALTER TABLE sch3.tmain DETACH PARTITION sch3.part2; +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain WITH (publish_via_partition_root); +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part2 FOR VALUES FROM (5) TO (10); + +-- Can't create foreign partition of published table with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch3.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; + +-- Can't attach foreign partition to published table with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_2(id int) SERVER fdw_server; +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); + +CREATE SCHEMA sch4; +CREATE TABLE sch4.tmain(id int) PARTITION BY RANGE(id); + +-- publication created with FOR TABLES IN SCHEMA +DROP PUBLICATION pub1; +CREATE PUBLICATION pub1 FOR TABLES IN SCHEMA sch4 WITH (publish_via_partition_root); + +-- Can't create foreign partition of published table with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch4.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; + +-- Can't attach foreign partition to published table with +-- publish_via_partition_root = true +ALTER TABLE sch4.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); +DROP PUBLICATION pub1; + +-- Test with publish_via_partition_root = false +-- Foreign partitions are skipped by default +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain; +CREATE PUBLICATION pub2 FOR TABLES IN SCHEMA sch3; +CREATE PUBLICATION pub3 FOR ALL TABLES; + +-- Create foreign partition of published table with +-- publish_via_partition_root = false +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch3.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; + +-- Attach foreign partition to published table +-- publish_via_partition_root = false +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); + +-- Check the published tables +SELECT pubname, tablename FROM pg_publication_tables WHERE schemaname in ('sch3', 'sch4') ORDER BY pubname, tablename; + +-- Can't alter publish_via_partition_root to true, if publication already have +-- foreign partition +ALTER PUBLICATION pub1 SET (publish_via_partition_root); + +DROP PUBLICATION pub1; +DROP PUBLICATION pub2; +DROP PUBLICATION pub3; +DROP SCHEMA sch3 CASCADE; +DROP SCHEMA sch4 CASCADE; +DROP SERVER fdw_server; +DROP FOREIGN DATA WRAPPER test_fdw; + RESET SESSION AUTHORIZATION; DROP ROLE regress_publication_user, regress_publication_user2; DROP ROLE regress_publication_user_dummy; -- 2.34.1 ^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: Restrict publishing of partitioned table with a foreign table as partition 2025-03-24 09:33 Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-03-24 15:47 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> 2025-03-28 09:43 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> @ 2025-03-28 11:04 ` Álvaro Herrera <[email protected]> 2025-04-01 14:48 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 0 siblings, 1 reply; 19+ messages in thread From: Álvaro Herrera @ 2025-03-28 11:04 UTC (permalink / raw) To: Shlok Kyal <[email protected]>; +Cc: vignesh C <[email protected]>; Sergey Tatarintsev <[email protected]>; [email protected] On 2025-Mar-28, Shlok Kyal wrote: > On Mon, 24 Mar 2025 at 21:17, Álvaro Herrera <[email protected]> wrote: > > Also, surely we should document this restriction in the SGML docs > > somewhere. > > I have added comment in create_publication.sgml Hmm, okay, but "We cannot" is not the style used in the documentation. In addition, I think this mechanism should be mentioned in logical-replication.sgml; currently there's a note in the Restrictions section about foreign tables, which should be expanded to explain this. -- Álvaro Herrera PostgreSQL Developer — https://www.EnterpriseDB.com/ ^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: Restrict publishing of partitioned table with a foreign table as partition 2025-03-24 09:33 Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-03-24 15:47 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> 2025-03-28 09:43 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-03-28 11:04 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> @ 2025-04-01 14:48 ` Shlok Kyal <[email protected]> 2025-04-04 05:06 ` Re: Restrict publishing of partitioned table with a foreign table as partition Sergey Tatarintsev <[email protected]> 0 siblings, 1 reply; 19+ messages in thread From: Shlok Kyal @ 2025-04-01 14:48 UTC (permalink / raw) To: Álvaro Herrera <[email protected]>; +Cc: vignesh C <[email protected]>; Sergey Tatarintsev <[email protected]>; [email protected] On Fri, 28 Mar 2025 at 16:35, Álvaro Herrera <[email protected]> wrote: > > On 2025-Mar-28, Shlok Kyal wrote: > > > On Mon, 24 Mar 2025 at 21:17, Álvaro Herrera <[email protected]> wrote: > > > > Also, surely we should document this restriction in the SGML docs > > > somewhere. > > > > I have added comment in create_publication.sgml > > Hmm, okay, but "We cannot" is not the style used in the documentation. > In addition, I think this mechanism should be mentioned in > logical-replication.sgml; currently there's a note in the Restrictions > section about foreign tables, which should be expanded to explain this. > I have modified the comment in create_publication.sgml and also added comment in the restrictions section of logical-replication.sgml. I have also added a more detailed explanation in comment of 'check_foreign_tables' I have attached the updated v11 patch. Thanks and Regards, Shlok Kyal Attachments: [application/octet-stream] v11-0001-Restrict-publishing-of-partitioned-table-with-fo.patch (30.0K, ../../CANhcyEXVPUAir9G1G1gT9k-xAMzhiV7GRX_fr1rkRhURNKDbxg@mail.gmail.com/2-v11-0001-Restrict-publishing-of-partitioned-table-with-fo.patch) download | inline diff: From 6fc197f6263db9c08af39db340ef7d7dd955c957 Mon Sep 17 00:00:00 2001 From: Shlok Kyal <[email protected]> Date: Fri, 28 Mar 2025 11:15:09 +0530 Subject: [PATCH v11] Restrict publishing of partitioned table with foreign partition Logical replication of foreign table is not supported and we throw an error in this case. But when create a publication on a partitioned table that has a foreign partition, the initial sync of such table is successful and we should avoid such cases. Current Behaviour in HEAD, when publication is created: 1. with publish_via_partition_root = true Root table is published and the initial data of foreign partitions are replicated. 2. with publish_via_partition_root = false and FOR ALL TABLES All leaf tables except foreign partitions are published. 3. with publish_via_partition_root = false and FOR TABLE/ FOR TABLES IN SCHEMA All leaf tables are published. Initial data of foreign partitions are replicated. With this patch we have following behaviour: 1. with publish_via_partition_root = true We throw an error when we try to publish a foreign partititon. Error is thrown when we try to create a publication on (or add to existing publication) a partitioned table with foreign partition, when try to create a foreign partition and when we try to attach foreign table (or a table with foreign partition) to existing published tables. 2. with publish_via_partition_root = false We skip publishing foreign partition which are part of the publication. This is done by avoid adding foreign partitions in pg_subscription_rel catalog table. We have introduced two functions 'check_partrel_has_foreign_table' and 'check_foreign_tables'. In 'check_partrel_has_foreign_table' we go through the child nodes of a partition and check if it has a foreign table. While doing so, we take a AccessShareLock on each partition table to avoid creation of foreign partition concurrently. In 'check_foreign_tables' if schema id is provided we check for each partitioned table in that schema if it has a foreign partition, or if schema id is not provided we check for each partitioned table in the database if it has a foreign partition. While doing so we take a ShareLock on pg_partitioned_table so no partition table is created concurrently after this check. --- doc/src/sgml/logical-replication.sgml | 10 +- doc/src/sgml/ref/create_publication.sgml | 5 + src/backend/catalog/pg_publication.c | 166 +++++++++++++++++++--- src/backend/commands/foreigncmds.c | 48 +++++++ src/backend/commands/publicationcmds.c | 43 ++++++ src/backend/commands/tablecmds.c | 57 ++++++++ src/include/catalog/pg_publication.h | 5 + src/test/regress/expected/publication.out | 89 ++++++++++++ src/test/regress/sql/publication.sql | 81 +++++++++++ 9 files changed, 486 insertions(+), 18 deletions(-) diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index f288c049a5c..ce60a1b391c 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -2154,7 +2154,15 @@ CONTEXT: processing remote data for replication origin "pg_16395" during "INSER <listitem> <para> - Replication is only supported by tables, including partitioned tables. + Replication is supported only for tables, including partitioned tables, + except when they contain foreign partitions. If + <literal>publish_via_partition_root</literal> is set to + <literal>true</literal> for a publication table with foreign partitions, or + if an attempt is made to replicate such a table, an error is thrown. + Additionally, when replicating a partitioned table where + <literal>publish_via_partition_root</literal> is set to + <literal>false</literal> and foreign partitions are present, all partitions + are replicated except the foreign partitions. Attempts to replicate other types of relations, such as views, materialized views, or foreign tables, will result in an error. </para> diff --git a/doc/src/sgml/ref/create_publication.sgml b/doc/src/sgml/ref/create_publication.sgml index 73f0c8d89fb..47216fc4789 100644 --- a/doc/src/sgml/ref/create_publication.sgml +++ b/doc/src/sgml/ref/create_publication.sgml @@ -257,6 +257,11 @@ CREATE PUBLICATION <replaceable class="parameter">name</replaceable> If this is enabled, <literal>TRUNCATE</literal> operations performed directly on partitions are not replicated. </para> + + <para> + If this is enabled, a foreign table or a partitioned table with a + foreign partition is not allowed in the publication. + </para> </listitem> </varlistentry> </variablelist></para> diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c index d6f94db5d99..77299b75cde 100644 --- a/src/backend/catalog/pg_publication.c +++ b/src/backend/catalog/pg_publication.c @@ -26,12 +26,14 @@ #include "catalog/partition.h" #include "catalog/pg_inherits.h" #include "catalog/pg_namespace.h" +#include "catalog/pg_partitioned_table.h" #include "catalog/pg_publication.h" #include "catalog/pg_publication_namespace.h" #include "catalog/pg_publication_rel.h" #include "catalog/pg_type.h" #include "commands/publicationcmds.h" #include "funcapi.h" +#include "partitioning/partdesc.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/catcache.h" @@ -53,7 +55,7 @@ typedef struct * error if not. */ static void -check_publication_add_relation(Relation targetrel) +check_publication_add_relation(Relation targetrel, Publication *pub) { /* Must be a regular or partitioned table */ if (RelationGetForm(targetrel)->relkind != RELKIND_RELATION && @@ -64,6 +66,18 @@ check_publication_add_relation(Relation targetrel) RelationGetRelationName(targetrel)), errdetail_relkind_not_supported(RelationGetForm(targetrel)->relkind))); + /* + * publish_via_root_partition cannot be true if it is a partitioned table + * and has any foreign partition. See check_foreign_tables for details. + */ + if (pub->pubviaroot && check_partrel_has_foreign_table(targetrel)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot set parameter \"%s\" to true for publication \"%s\"", + "publish_via_partition_root", pub->name), + errdetail("partition table \"%s\" in publication contains a foreign partition", + RelationGetRelationName(targetrel)))); + /* Can't be system table */ if (IsCatalogRelation(targetrel)) ereport(ERROR, @@ -304,7 +318,7 @@ check_and_fetch_column_list(Publication *pub, Oid relid, MemoryContext mcxt, /* * Gets the relations based on the publication partition option for a specified - * relation. + * relation. Foreign tables are not included. */ List * GetPubPartitionOptionRelations(List *result, PublicationPartOpt pub_partopt, @@ -313,25 +327,21 @@ GetPubPartitionOptionRelations(List *result, PublicationPartOpt pub_partopt, if (get_rel_relkind(relid) == RELKIND_PARTITIONED_TABLE && pub_partopt != PUBLICATION_PART_ROOT) { - List *all_parts = find_all_inheritors(relid, NoLock, - NULL); + List *all_parts = find_all_inheritors(relid, NoLock, NULL); - if (pub_partopt == PUBLICATION_PART_ALL) - result = list_concat(result, all_parts); - else if (pub_partopt == PUBLICATION_PART_LEAF) + foreach_oid(partOid, all_parts) { - ListCell *lc; + char relkind = get_rel_relkind(partOid); - foreach(lc, all_parts) - { - Oid partOid = lfirst_oid(lc); + if (relkind == RELKIND_FOREIGN_TABLE) + continue; - if (get_rel_relkind(partOid) != RELKIND_PARTITIONED_TABLE) - result = lappend_oid(result, partOid); - } + if (pub_partopt == PUBLICATION_PART_LEAF && + relkind == RELKIND_PARTITIONED_TABLE) + continue; + + result = lappend_oid(result, partOid); } - else - Assert(false); } else result = lappend_oid(result, relid); @@ -463,7 +473,7 @@ publication_add_relation(Oid pubid, PublicationRelInfo *pri, RelationGetRelationName(targetrel), pub->name))); } - check_publication_add_relation(targetrel); + check_publication_add_relation(targetrel, pub); /* Validate and translate column names into a Bitmapset of attnums. */ attnums = pub_collist_validate(pri->relation, pri->columns); @@ -703,6 +713,13 @@ publication_add_schema(Oid pubid, Oid schemaid, bool if_not_exists) check_publication_add_schema(schemaid); + /* + * If publish_via_partition_root is true, check if schema has any foreign + * partition + */ + if (pub->pubviaroot) + check_foreign_tables(schemaid, pub->name); + /* Form a tuple */ memset(values, 0, sizeof(values)); memset(nulls, false, sizeof(nulls)); @@ -1332,3 +1349,118 @@ pg_get_publication_tables(PG_FUNCTION_ARGS) SRF_RETURN_DONE(funcctx); } + +/* Check if a partitioned table has a foreign partition */ +bool +check_partrel_has_foreign_table(Relation rel) +{ + bool has_foreign_tbl = false; + + if (RelationGetForm(rel)->relkind == RELKIND_PARTITIONED_TABLE) + { + PartitionDesc pd = RelationGetPartitionDesc(rel, true); + + for (int i = 0; i < pd->nparts; i++) + { + Relation childrel = table_open(pd->oids[i], AccessShareLock); + + if (RelationGetForm(childrel)->relkind == RELKIND_FOREIGN_TABLE) + has_foreign_tbl = true; + else + has_foreign_tbl = check_partrel_has_foreign_table(childrel); + + table_close(childrel, NoLock); + + if (has_foreign_tbl) + break; + } + } + + return has_foreign_tbl; +} + +/* + * Protect against including foreign tables that are partitions of partitioned + * tables published by the given publication when publish_via_root_partition is + * true. This will not work correctly as the initial data from the foreign + * table can be replicated by the tablesync worker even though replication of + * foreign table is not supported because when publish_via_root_partition is + * true, the root table is included in pg_subscription_rel catalog table and + * tablesync worker cannot distinguish data from foreign partition. So we + * disallow the case here and in all DDL commands that would end up creating + * such a case indirectly. + * + * When publish_via_root_partition is set to false, leaf partitions are included + * in pg_subscription_rel catalog table. So, when we include a partition table + * with foreign partition in a publication, we skip including foreign partitions + * to pg_subscription_rel catalog table. So, the foreign partitions are not + * replicated. + * + * + * check_foreign_tables + * + * If valid schemaid provided, check if the schema has a partition table which + * has a foreign partition. The partition tables in a schema can have partitions + * in other schema. We also need to check if such partitions are foreign + * partition. + * + * If valid schemaid is not provided, we get all partition tables and check if + * it has any foreign partition. We take a lock on partition tables so no new + * foreign partitions are added concurrently. + * + * We take a ShareLock on pg_partitioned_table to restrict addition of new + * partitioned table which may contain a foreign partition while publication is + * being created. + */ +void +check_foreign_tables(Oid schemaid, char *pubname) +{ + Relation classRel; + Relation partRel; + ScanKeyData key[3]; + int keycount = 0; + TableScanDesc scan; + HeapTuple tuple; + + classRel = table_open(RelationRelationId, AccessShareLock); + partRel = table_open(PartitionedRelationId, ShareLock); + + /* Get the root nodes of partitioned table */ + ScanKeyInit(&key[keycount++], + Anum_pg_class_relkind, + BTEqualStrategyNumber, F_CHAREQ, + CharGetDatum(RELKIND_PARTITIONED_TABLE)); + + ScanKeyInit(&key[keycount++], + Anum_pg_class_relispartition, + BTEqualStrategyNumber, F_BOOLEQ, + BoolGetDatum(false)); + + /* If schema id is provided check partitioned table in that schema */ + if (OidIsValid(schemaid)) + ScanKeyInit(&key[keycount++], + Anum_pg_class_relnamespace, + BTEqualStrategyNumber, F_OIDEQ, + schemaid); + + scan = table_beginscan_catalog(classRel, keycount, key); + while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) + { + Form_pg_class relForm = (Form_pg_class) GETSTRUCT(tuple); + Relation pubrel = table_open(relForm->oid, AccessShareLock); + + if (check_partrel_has_foreign_table(pubrel)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot set parameter \"%s\" to true for publication \"%s\"", + "publish_via_partition_root", pubname), + errdetail("partition table \"%s\" in publication contains a foreign partition", + get_rel_name(relForm->oid)))); + + table_close(pubrel, NoLock); + } + + table_endscan(scan); + table_close(classRel, AccessShareLock); + table_close(partRel, NoLock); +} diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c index c14e038d54f..abc23baa59e 100644 --- a/src/backend/commands/foreigncmds.c +++ b/src/backend/commands/foreigncmds.c @@ -21,6 +21,7 @@ #include "catalog/dependency.h" #include "catalog/indexing.h" #include "catalog/objectaccess.h" +#include "catalog/partition.h" #include "catalog/pg_foreign_data_wrapper.h" #include "catalog/pg_foreign_server.h" #include "catalog/pg_foreign_table.h" @@ -1423,6 +1424,53 @@ CreateForeignTable(CreateForeignTableStmt *stmt, Oid relid) ftrel = table_open(ForeignTableRelationId, RowExclusiveLock); + /* + * Check if it is a foreign partition and the partitioned table is not + * published or published with publish_via_partition_root option as false. + * See check_foreign_tables for details. + */ + if (stmt->base.partbound != NULL) + { + RangeVar *root = linitial_node(RangeVar, stmt->base.inhRelations); + Relation rootrel = table_openrv(root, AccessShareLock); + + if (RelationGetForm(rootrel)->relkind == RELKIND_PARTITIONED_TABLE) + { + Oid schemaid = RelationGetNamespace(rootrel); + List *puboids = GetRelationPublications(rootrel->rd_id); + List *ancestors; + + puboids = list_concat_unique_oid(puboids, GetAllTablesPublications()); + puboids = list_concat_unique_oid(puboids, GetSchemaPublications(schemaid)); + ancestors = get_partition_ancestors(rootrel->rd_id); + + foreach_oid(ancestor, ancestors) + { + puboids = list_concat_unique_oid(puboids, + GetRelationPublications(ancestor)); + schemaid = get_rel_namespace(ancestor); + puboids = list_concat_unique_oid(puboids, + GetSchemaPublications(schemaid)); + } + list_free(ancestors); + + foreach_oid(puboid, puboids) + { + Publication *pub = GetPublication(puboid); + + if (pub->pubviaroot) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot create table foreign partition \"%s\"", + get_rel_name(relid)), + errdetail("partition table \"%s\" is published with option publish_via_partition_root", + RelationGetRelationName(rootrel)))); + } + } + + table_close(rootrel, AccessShareLock); + } + /* * For now the owner cannot be specified on create. Use effective user ID. */ diff --git a/src/backend/commands/publicationcmds.c b/src/backend/commands/publicationcmds.c index 0b23d94c38e..e5fa1bf5b09 100644 --- a/src/backend/commands/publicationcmds.c +++ b/src/backend/commands/publicationcmds.c @@ -915,6 +915,10 @@ CreatePublication(ParseState *pstate, CreatePublicationStmt *stmt) /* Associate objects with the publication. */ if (stmt->for_all_tables) { + /* Check if any partitioned table has foreign partition */ + if (publish_via_partition_root) + check_foreign_tables(InvalidOid, stmt->pubname); + /* Invalidate relcache so that publication info is rebuilt. */ CacheInvalidateRelcacheAll(); } @@ -1080,6 +1084,45 @@ AlterPublicationOptions(ParseState *pstate, AlterPublicationStmt *stmt, } } + /* + * If publish_via_partition_root is set to true, check if the publication + * already have any foreign partition. See check_foreign_tables for details. + */ + if (publish_via_partition_root_given && publish_via_partition_root) + { + List *schemaoids = NIL; + List *relids = NIL; + + char *pubname = stmt->pubname; + + if (pubform->puballtables) + check_foreign_tables(InvalidOid, pubname); + + schemaoids = GetPublicationSchemas(pubform->oid); + + foreach_oid(schemaoid, schemaoids) + check_foreign_tables(schemaoid, pubname); + + relids = GetPublicationRelations(pubform->oid, PUBLICATION_PART_ROOT); + + foreach_oid(relid, relids) + { + Relation pubrel = table_open(relid, AccessShareLock); + + if (check_partrel_has_foreign_table(pubrel)) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot set parameter \"%s\" to true for publication \"%s\"", + "publish_via_partition_root", pubname), + errdetail("partition table \"%s\" in publication contains a foreign partition", + get_rel_name(relid)))); + } + + table_close(pubrel, NoLock); + } + } + /* Everything ok, form a new tuple. */ memset(values, 0, sizeof(values)); memset(nulls, false, sizeof(nulls)); diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 10624353b0a..beee7010481 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -19619,6 +19619,10 @@ QueuePartitionConstraintValidation(List **wqueue, Relation scanrel, * ALTER TABLE <name> ATTACH PARTITION <partition-name> FOR VALUES * * Return the address of the newly attached partition. + * + * If the would-be partition is a foreign table or a partitioned table with + * foreign partition, verify that the partitioned table is not in a publication + * with publish_via_partition_root=true. See check_foreign_tables for details. */ static ObjectAddress ATExecAttachPartition(List **wqueue, Relation rel, PartitionCmd *cmd, @@ -19763,6 +19767,59 @@ ATExecAttachPartition(List **wqueue, Relation rel, PartitionCmd *cmd, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot attach temporary relation of another session as partition"))); + /* + * Check if attachrel is a foreign table or a partitioned table with + * foreign partition and rel is not part of publication with option + * publish_via_partition_root as true. + */ + if (attachrel->rd_rel->relkind == RELKIND_FOREIGN_TABLE || + check_partrel_has_foreign_table(attachrel)) + { + Oid schemaid = RelationGetNamespace(rel); + List *puboids = GetRelationPublications(rel->rd_id); + List *ancestors; + char *relname = get_rel_name(rel->rd_id); + char *attachrelname = get_rel_name(attachrel->rd_id); + + puboids = list_concat_unique_oid(puboids, GetAllTablesPublications()); + puboids = list_concat_unique_oid(puboids, GetSchemaPublications(schemaid)); + ancestors = get_partition_ancestors(rel->rd_id); + + foreach_oid(ancestor, ancestors) + { + puboids = list_concat_unique_oid(puboids, + GetRelationPublications(ancestor)); + schemaid = get_rel_namespace(ancestor); + puboids = list_concat_unique_oid(puboids, + GetSchemaPublications(schemaid)); + } + + list_free(ancestors); + + foreach_oid(puboid, puboids) + { + Publication *pub = GetPublication(puboid); + + if (pub->pubviaroot) + { + if (attachrel->rd_rel->relkind == RELKIND_FOREIGN_TABLE) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot attach foreign table \"%s\" to partition table \"%s\"", + attachrelname, relname), + errdetail("partition table \"%s\" is published with option publish_via_partition_root", + relname))); + else + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot attach table \"%s\" with foreign partition to partition table \"%s\"", + attachrelname, relname), + errdetail("partition table \"%s\" is published with option publish_via_partition_root", + relname))); + } + } + } + /* * Check if attachrel has any identity columns or any columns that aren't * in the parent. diff --git a/src/include/catalog/pg_publication.h b/src/include/catalog/pg_publication.h index 48c7d1a8615..e2919da2541 100644 --- a/src/include/catalog/pg_publication.h +++ b/src/include/catalog/pg_publication.h @@ -19,6 +19,7 @@ #include "catalog/genbki.h" #include "catalog/objectaddress.h" +#include "catalog/pg_class.h" #include "catalog/pg_publication_d.h" /* IWYU pragma: export */ /* ---------------- @@ -191,4 +192,8 @@ extern Bitmapset *pub_collist_to_bitmapset(Bitmapset *columns, Datum pubcols, extern Bitmapset *pub_form_cols_map(Relation relation, PublishGencolsType include_gencols_type); +extern bool check_partrel_has_foreign_table(Relation rel); + +extern void check_foreign_tables(Oid schemaid, char *pubname); + #endif /* PG_PUBLICATION_H */ diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out index 4de96c04f9d..6a0c245bd60 100644 --- a/src/test/regress/expected/publication.out +++ b/src/test/regress/expected/publication.out @@ -1924,6 +1924,95 @@ DROP PUBLICATION pub1; DROP PUBLICATION pub2; DROP TABLE gencols; RESET client_min_messages; +-- ====================================================== +-- Test when foreign table is a partition of a partitioned table on which +-- publication is created +SET client_min_messages = 'ERROR'; +CREATE FOREIGN DATA WRAPPER test_fdw; +CREATE SERVER fdw_server FOREIGN DATA WRAPPER test_fdw; +CREATE SCHEMA sch3; +CREATE TABLE sch3.tmain(id int) PARTITION BY RANGE(id); +CREATE TABLE sch3.part1 PARTITION OF sch3.tmain FOR VALUES FROM (0) TO (5); +CREATE TABLE sch3.part2(id int) PARTITION BY RANGE(id); +CREATE FOREIGN TABLE sch3.part2_1 PARTITION OF sch3.part2 FOR VALUES FROM (5) TO (10) SERVER fdw_server; +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part2 FOR VALUES FROM (5) TO (10); +-- Can't create publications with publish_via_partition_root = true, if table +-- has a foreign partition +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain WITH (publish_via_partition_root); +ERROR: cannot set parameter "publish_via_partition_root" to true for publication "pub1" +DETAIL: partition table "tmain" in publication contains a foreign partition +CREATE PUBLICATION pub1 FOR TABLES IN SCHEMA sch3 WITH (publish_via_partition_root); +ERROR: cannot set parameter "publish_via_partition_root" to true for publication "pub1" +DETAIL: partition table "tmain" in publication contains a foreign partition +CREATE PUBLICATION pub1 FOR ALL TABLES WITH (publish_via_partition_root); +ERROR: cannot set parameter "publish_via_partition_root" to true for publication "pub1" +DETAIL: partition table "tmain" in publication contains a foreign partition +-- Test when a partitioned table with foreign table as a partition is attached +-- to partitioned table which is already published +ALTER TABLE sch3.tmain DETACH PARTITION sch3.part2; +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain WITH (publish_via_partition_root); +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part2 FOR VALUES FROM (5) TO (10); +ERROR: cannot attach table "part2" with foreign partition to partition table "tmain" +DETAIL: partition table "tmain" is published with option publish_via_partition_root +-- Can't create foreign partition of published table with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch3.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; +ERROR: cannot create table foreign partition "part3_1" +DETAIL: partition table "tmain" is published with option publish_via_partition_root +-- Can't attach foreign partition to published table with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_2(id int) SERVER fdw_server; +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); +ERROR: cannot attach foreign table "part3_2" to partition table "tmain" +DETAIL: partition table "tmain" is published with option publish_via_partition_root +CREATE SCHEMA sch4; +CREATE TABLE sch4.tmain(id int) PARTITION BY RANGE(id); +-- publication created with FOR TABLES IN SCHEMA +DROP PUBLICATION pub1; +CREATE PUBLICATION pub1 FOR TABLES IN SCHEMA sch4 WITH (publish_via_partition_root); +-- Can't create foreign partition of published table with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch4.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; +ERROR: cannot create table foreign partition "part3_1" +DETAIL: partition table "tmain" is published with option publish_via_partition_root +-- Can't attach foreign partition to published table with +-- publish_via_partition_root = true +ALTER TABLE sch4.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); +ERROR: cannot attach foreign table "part3_2" to partition table "tmain" +DETAIL: partition table "tmain" is published with option publish_via_partition_root +DROP PUBLICATION pub1; +-- Test with publish_via_partition_root = false +-- Foreign partitions are skipped by default +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain; +CREATE PUBLICATION pub2 FOR TABLES IN SCHEMA sch3; +CREATE PUBLICATION pub3 FOR ALL TABLES; +-- Create foreign partition of published table with +-- publish_via_partition_root = false +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch3.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; +-- Attach foreign partition to published table +-- publish_via_partition_root = false +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); +-- Check the published tables +SELECT pubname, tablename FROM pg_publication_tables WHERE schemaname in ('sch3', 'sch4') ORDER BY pubname, tablename; + pubname | tablename +---------+----------- + pub1 | part1 + pub2 | part1 + pub3 | part1 +(3 rows) + +-- Can't alter publish_via_partition_root to true, if publication already have +-- foreign partition +ALTER PUBLICATION pub1 SET (publish_via_partition_root); +ERROR: cannot set parameter "publish_via_partition_root" to true for publication "pub1" +DETAIL: partition table "tmain" in publication contains a foreign partition +DROP PUBLICATION pub1; +DROP PUBLICATION pub2; +DROP PUBLICATION pub3; +DROP SCHEMA sch3 CASCADE; +DROP SCHEMA sch4 CASCADE; +DROP SERVER fdw_server; +DROP FOREIGN DATA WRAPPER test_fdw; RESET SESSION AUTHORIZATION; DROP ROLE regress_publication_user, regress_publication_user2; DROP ROLE regress_publication_user_dummy; diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql index 68001de4000..49c9d98b668 100644 --- a/src/test/regress/sql/publication.sql +++ b/src/test/regress/sql/publication.sql @@ -1223,6 +1223,87 @@ DROP PUBLICATION pub2; DROP TABLE gencols; RESET client_min_messages; +-- ====================================================== + +-- Test when foreign table is a partition of a partitioned table on which +-- publication is created +SET client_min_messages = 'ERROR'; +CREATE FOREIGN DATA WRAPPER test_fdw; +CREATE SERVER fdw_server FOREIGN DATA WRAPPER test_fdw; + +CREATE SCHEMA sch3; +CREATE TABLE sch3.tmain(id int) PARTITION BY RANGE(id); +CREATE TABLE sch3.part1 PARTITION OF sch3.tmain FOR VALUES FROM (0) TO (5); +CREATE TABLE sch3.part2(id int) PARTITION BY RANGE(id); +CREATE FOREIGN TABLE sch3.part2_1 PARTITION OF sch3.part2 FOR VALUES FROM (5) TO (10) SERVER fdw_server; +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part2 FOR VALUES FROM (5) TO (10); + +-- Can't create publications with publish_via_partition_root = true, if table +-- has a foreign partition +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain WITH (publish_via_partition_root); +CREATE PUBLICATION pub1 FOR TABLES IN SCHEMA sch3 WITH (publish_via_partition_root); +CREATE PUBLICATION pub1 FOR ALL TABLES WITH (publish_via_partition_root); + +-- Test when a partitioned table with foreign table as a partition is attached +-- to partitioned table which is already published +ALTER TABLE sch3.tmain DETACH PARTITION sch3.part2; +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain WITH (publish_via_partition_root); +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part2 FOR VALUES FROM (5) TO (10); + +-- Can't create foreign partition of published table with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch3.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; + +-- Can't attach foreign partition to published table with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_2(id int) SERVER fdw_server; +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); + +CREATE SCHEMA sch4; +CREATE TABLE sch4.tmain(id int) PARTITION BY RANGE(id); + +-- publication created with FOR TABLES IN SCHEMA +DROP PUBLICATION pub1; +CREATE PUBLICATION pub1 FOR TABLES IN SCHEMA sch4 WITH (publish_via_partition_root); + +-- Can't create foreign partition of published table with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch4.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; + +-- Can't attach foreign partition to published table with +-- publish_via_partition_root = true +ALTER TABLE sch4.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); +DROP PUBLICATION pub1; + +-- Test with publish_via_partition_root = false +-- Foreign partitions are skipped by default +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain; +CREATE PUBLICATION pub2 FOR TABLES IN SCHEMA sch3; +CREATE PUBLICATION pub3 FOR ALL TABLES; + +-- Create foreign partition of published table with +-- publish_via_partition_root = false +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch3.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; + +-- Attach foreign partition to published table +-- publish_via_partition_root = false +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); + +-- Check the published tables +SELECT pubname, tablename FROM pg_publication_tables WHERE schemaname in ('sch3', 'sch4') ORDER BY pubname, tablename; + +-- Can't alter publish_via_partition_root to true, if publication already have +-- foreign partition +ALTER PUBLICATION pub1 SET (publish_via_partition_root); + +DROP PUBLICATION pub1; +DROP PUBLICATION pub2; +DROP PUBLICATION pub3; +DROP SCHEMA sch3 CASCADE; +DROP SCHEMA sch4 CASCADE; +DROP SERVER fdw_server; +DROP FOREIGN DATA WRAPPER test_fdw; + RESET SESSION AUTHORIZATION; DROP ROLE regress_publication_user, regress_publication_user2; DROP ROLE regress_publication_user_dummy; -- 2.34.1 ^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: Restrict publishing of partitioned table with a foreign table as partition 2025-03-24 09:33 Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-03-24 15:47 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> 2025-03-28 09:43 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-03-28 11:04 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> 2025-04-01 14:48 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> @ 2025-04-04 05:06 ` Sergey Tatarintsev <[email protected]> 2025-04-04 09:14 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 0 siblings, 1 reply; 19+ messages in thread From: Sergey Tatarintsev @ 2025-04-04 05:06 UTC (permalink / raw) To: Shlok Kyal <[email protected]>; Álvaro Herrera <[email protected]>; +Cc: vignesh C <[email protected]>; [email protected] 01.04.2025 21:48, Shlok Kyal пишет: > On Fri, 28 Mar 2025 at 16:35, Álvaro Herrera <[email protected]> wrote: >> On 2025-Mar-28, Shlok Kyal wrote: >> >>> On Mon, 24 Mar 2025 at 21:17, Álvaro Herrera <[email protected]> wrote: >>>> Also, surely we should document this restriction in the SGML docs >>>> somewhere. >>> I have added comment in create_publication.sgml >> Hmm, okay, but "We cannot" is not the style used in the documentation. >> In addition, I think this mechanism should be mentioned in >> logical-replication.sgml; currently there's a note in the Restrictions >> section about foreign tables, which should be expanded to explain this. >> > I have modified the comment in create_publication.sgml and also added > comment in the restrictions section of logical-replication.sgml. > I have also added a more detailed explanation in comment of > 'check_foreign_tables' > > I have attached the updated v11 patch. > > > Thanks and Regards, > Shlok Kyal Hi! I looked at the latest version of the patch, and think that we should free puboids list here: diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 6a128f7bd4e..4254654cc24 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -20122,6 +20122,7 @@ ATExecAttachPartition(List **wqueue, Relation rel, PartitionCmd *cmd, relname))); } } + list_free(puboids); } /* -- With best regards, Sergey Tatarintsev, PostgresPro ^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: Restrict publishing of partitioned table with a foreign table as partition 2025-03-24 09:33 Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-03-24 15:47 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> 2025-03-28 09:43 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-03-28 11:04 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> 2025-04-01 14:48 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-04-04 05:06 ` Re: Restrict publishing of partitioned table with a foreign table as partition Sergey Tatarintsev <[email protected]> @ 2025-04-04 09:14 ` Shlok Kyal <[email protected]> 0 siblings, 0 replies; 19+ messages in thread From: Shlok Kyal @ 2025-04-04 09:14 UTC (permalink / raw) To: Sergey Tatarintsev <[email protected]>; +Cc: Álvaro Herrera <[email protected]>; vignesh C <[email protected]>; [email protected] On Fri, 4 Apr 2025 at 10:36, Sergey Tatarintsev <[email protected]> wrote: > > 01.04.2025 21:48, Shlok Kyal пишет: > > On Fri, 28 Mar 2025 at 16:35, Álvaro Herrera <[email protected]> wrote: > >> On 2025-Mar-28, Shlok Kyal wrote: > >> > >>> On Mon, 24 Mar 2025 at 21:17, Álvaro Herrera <[email protected]> wrote: > >>>> Also, surely we should document this restriction in the SGML docs > >>>> somewhere. > >>> I have added comment in create_publication.sgml > >> Hmm, okay, but "We cannot" is not the style used in the documentation. > >> In addition, I think this mechanism should be mentioned in > >> logical-replication.sgml; currently there's a note in the Restrictions > >> section about foreign tables, which should be expanded to explain this. > >> > > I have modified the comment in create_publication.sgml and also added > > comment in the restrictions section of logical-replication.sgml. > > I have also added a more detailed explanation in comment of > > 'check_foreign_tables' > > > > I have attached the updated v11 patch. > > > > > > Thanks and Regards, > > Shlok Kyal > > Hi! > > I looked at the latest version of the patch, and think that we should > free puboids list here: > > diff --git a/src/backend/commands/tablecmds.c > b/src/backend/commands/tablecmds.c > index 6a128f7bd4e..4254654cc24 100644 > --- a/src/backend/commands/tablecmds.c > +++ b/src/backend/commands/tablecmds.c > @@ -20122,6 +20122,7 @@ ATExecAttachPartition(List **wqueue, Relation > rel, PartitionCmd *cmd, > relname))); > } > } > + list_free(puboids); > } > > /* > Hi Sergey, Thanks for reviewing the patch. I have fixed the comment. I also found other places where we should free the relids, schemaoids, and puboids list. I have added changes for those as well. I have attached the updated patch. Thanks and Regards, Shlok Kyal Attachments: [application/octet-stream] v12-0001-Restrict-publishing-of-partitioned-table-with-fo.patch (30.1K, ../../CANhcyEXSOuv7=ueqU3ZqZCsFGb1+EEpZVQAUkz1uvUpJTUOjqg@mail.gmail.com/2-v12-0001-Restrict-publishing-of-partitioned-table-with-fo.patch) download | inline diff: From 7ef4b3095c9a0b308192898302e8415ba27d60e6 Mon Sep 17 00:00:00 2001 From: Shlok Kyal <[email protected]> Date: Fri, 28 Mar 2025 11:15:09 +0530 Subject: [PATCH v12] Restrict publishing of partitioned table with foreign partition Logical replication of foreign table is not supported and we throw an error in this case. But when create a publication on a partitioned table that has a foreign partition, the initial sync of such table is successful and we should avoid such cases. Current Behaviour in HEAD, when publication is created: 1. with publish_via_partition_root = true Root table is published and the initial data of foreign partitions are replicated. 2. with publish_via_partition_root = false and FOR ALL TABLES All leaf tables except foreign partitions are published. 3. with publish_via_partition_root = false and FOR TABLE/ FOR TABLES IN SCHEMA All leaf tables are published. Initial data of foreign partitions are replicated. With this patch we have following behaviour: 1. with publish_via_partition_root = true We throw an error when we try to publish a foreign partititon. Error is thrown when we try to create a publication on (or add to existing publication) a partitioned table with foreign partition, when try to create a foreign partition and when we try to attach foreign table (or a table with foreign partition) to existing published tables. 2. with publish_via_partition_root = false We skip publishing foreign partition which are part of the publication. This is done by avoid adding foreign partitions in pg_subscription_rel catalog table. We have introduced two functions 'check_partrel_has_foreign_table' and 'check_foreign_tables'. In 'check_partrel_has_foreign_table' we go through the child nodes of a partition and check if it has a foreign table. While doing so, we take a AccessShareLock on each partition table to avoid creation of foreign partition concurrently. In 'check_foreign_tables' if schema id is provided we check for each partitioned table in that schema if it has a foreign partition, or if schema id is not provided we check for each partitioned table in the database if it has a foreign partition. While doing so we take a ShareLock on pg_partitioned_table so no partition table is created concurrently after this check. --- doc/src/sgml/logical-replication.sgml | 10 +- doc/src/sgml/ref/create_publication.sgml | 5 + src/backend/catalog/pg_publication.c | 166 +++++++++++++++++++--- src/backend/commands/foreigncmds.c | 50 +++++++ src/backend/commands/publicationcmds.c | 47 ++++++ src/backend/commands/tablecmds.c | 59 ++++++++ src/include/catalog/pg_publication.h | 5 + src/test/regress/expected/publication.out | 89 ++++++++++++ src/test/regress/sql/publication.sql | 81 +++++++++++ 9 files changed, 494 insertions(+), 18 deletions(-) diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index f288c049a5c..ce60a1b391c 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -2154,7 +2154,15 @@ CONTEXT: processing remote data for replication origin "pg_16395" during "INSER <listitem> <para> - Replication is only supported by tables, including partitioned tables. + Replication is supported only for tables, including partitioned tables, + except when they contain foreign partitions. If + <literal>publish_via_partition_root</literal> is set to + <literal>true</literal> for a publication table with foreign partitions, or + if an attempt is made to replicate such a table, an error is thrown. + Additionally, when replicating a partitioned table where + <literal>publish_via_partition_root</literal> is set to + <literal>false</literal> and foreign partitions are present, all partitions + are replicated except the foreign partitions. Attempts to replicate other types of relations, such as views, materialized views, or foreign tables, will result in an error. </para> diff --git a/doc/src/sgml/ref/create_publication.sgml b/doc/src/sgml/ref/create_publication.sgml index 73f0c8d89fb..47216fc4789 100644 --- a/doc/src/sgml/ref/create_publication.sgml +++ b/doc/src/sgml/ref/create_publication.sgml @@ -257,6 +257,11 @@ CREATE PUBLICATION <replaceable class="parameter">name</replaceable> If this is enabled, <literal>TRUNCATE</literal> operations performed directly on partitions are not replicated. </para> + + <para> + If this is enabled, a foreign table or a partitioned table with a + foreign partition is not allowed in the publication. + </para> </listitem> </varlistentry> </variablelist></para> diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c index d6f94db5d99..77299b75cde 100644 --- a/src/backend/catalog/pg_publication.c +++ b/src/backend/catalog/pg_publication.c @@ -26,12 +26,14 @@ #include "catalog/partition.h" #include "catalog/pg_inherits.h" #include "catalog/pg_namespace.h" +#include "catalog/pg_partitioned_table.h" #include "catalog/pg_publication.h" #include "catalog/pg_publication_namespace.h" #include "catalog/pg_publication_rel.h" #include "catalog/pg_type.h" #include "commands/publicationcmds.h" #include "funcapi.h" +#include "partitioning/partdesc.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/catcache.h" @@ -53,7 +55,7 @@ typedef struct * error if not. */ static void -check_publication_add_relation(Relation targetrel) +check_publication_add_relation(Relation targetrel, Publication *pub) { /* Must be a regular or partitioned table */ if (RelationGetForm(targetrel)->relkind != RELKIND_RELATION && @@ -64,6 +66,18 @@ check_publication_add_relation(Relation targetrel) RelationGetRelationName(targetrel)), errdetail_relkind_not_supported(RelationGetForm(targetrel)->relkind))); + /* + * publish_via_root_partition cannot be true if it is a partitioned table + * and has any foreign partition. See check_foreign_tables for details. + */ + if (pub->pubviaroot && check_partrel_has_foreign_table(targetrel)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot set parameter \"%s\" to true for publication \"%s\"", + "publish_via_partition_root", pub->name), + errdetail("partition table \"%s\" in publication contains a foreign partition", + RelationGetRelationName(targetrel)))); + /* Can't be system table */ if (IsCatalogRelation(targetrel)) ereport(ERROR, @@ -304,7 +318,7 @@ check_and_fetch_column_list(Publication *pub, Oid relid, MemoryContext mcxt, /* * Gets the relations based on the publication partition option for a specified - * relation. + * relation. Foreign tables are not included. */ List * GetPubPartitionOptionRelations(List *result, PublicationPartOpt pub_partopt, @@ -313,25 +327,21 @@ GetPubPartitionOptionRelations(List *result, PublicationPartOpt pub_partopt, if (get_rel_relkind(relid) == RELKIND_PARTITIONED_TABLE && pub_partopt != PUBLICATION_PART_ROOT) { - List *all_parts = find_all_inheritors(relid, NoLock, - NULL); + List *all_parts = find_all_inheritors(relid, NoLock, NULL); - if (pub_partopt == PUBLICATION_PART_ALL) - result = list_concat(result, all_parts); - else if (pub_partopt == PUBLICATION_PART_LEAF) + foreach_oid(partOid, all_parts) { - ListCell *lc; + char relkind = get_rel_relkind(partOid); - foreach(lc, all_parts) - { - Oid partOid = lfirst_oid(lc); + if (relkind == RELKIND_FOREIGN_TABLE) + continue; - if (get_rel_relkind(partOid) != RELKIND_PARTITIONED_TABLE) - result = lappend_oid(result, partOid); - } + if (pub_partopt == PUBLICATION_PART_LEAF && + relkind == RELKIND_PARTITIONED_TABLE) + continue; + + result = lappend_oid(result, partOid); } - else - Assert(false); } else result = lappend_oid(result, relid); @@ -463,7 +473,7 @@ publication_add_relation(Oid pubid, PublicationRelInfo *pri, RelationGetRelationName(targetrel), pub->name))); } - check_publication_add_relation(targetrel); + check_publication_add_relation(targetrel, pub); /* Validate and translate column names into a Bitmapset of attnums. */ attnums = pub_collist_validate(pri->relation, pri->columns); @@ -703,6 +713,13 @@ publication_add_schema(Oid pubid, Oid schemaid, bool if_not_exists) check_publication_add_schema(schemaid); + /* + * If publish_via_partition_root is true, check if schema has any foreign + * partition + */ + if (pub->pubviaroot) + check_foreign_tables(schemaid, pub->name); + /* Form a tuple */ memset(values, 0, sizeof(values)); memset(nulls, false, sizeof(nulls)); @@ -1332,3 +1349,118 @@ pg_get_publication_tables(PG_FUNCTION_ARGS) SRF_RETURN_DONE(funcctx); } + +/* Check if a partitioned table has a foreign partition */ +bool +check_partrel_has_foreign_table(Relation rel) +{ + bool has_foreign_tbl = false; + + if (RelationGetForm(rel)->relkind == RELKIND_PARTITIONED_TABLE) + { + PartitionDesc pd = RelationGetPartitionDesc(rel, true); + + for (int i = 0; i < pd->nparts; i++) + { + Relation childrel = table_open(pd->oids[i], AccessShareLock); + + if (RelationGetForm(childrel)->relkind == RELKIND_FOREIGN_TABLE) + has_foreign_tbl = true; + else + has_foreign_tbl = check_partrel_has_foreign_table(childrel); + + table_close(childrel, NoLock); + + if (has_foreign_tbl) + break; + } + } + + return has_foreign_tbl; +} + +/* + * Protect against including foreign tables that are partitions of partitioned + * tables published by the given publication when publish_via_root_partition is + * true. This will not work correctly as the initial data from the foreign + * table can be replicated by the tablesync worker even though replication of + * foreign table is not supported because when publish_via_root_partition is + * true, the root table is included in pg_subscription_rel catalog table and + * tablesync worker cannot distinguish data from foreign partition. So we + * disallow the case here and in all DDL commands that would end up creating + * such a case indirectly. + * + * When publish_via_root_partition is set to false, leaf partitions are included + * in pg_subscription_rel catalog table. So, when we include a partition table + * with foreign partition in a publication, we skip including foreign partitions + * to pg_subscription_rel catalog table. So, the foreign partitions are not + * replicated. + * + * + * check_foreign_tables + * + * If valid schemaid provided, check if the schema has a partition table which + * has a foreign partition. The partition tables in a schema can have partitions + * in other schema. We also need to check if such partitions are foreign + * partition. + * + * If valid schemaid is not provided, we get all partition tables and check if + * it has any foreign partition. We take a lock on partition tables so no new + * foreign partitions are added concurrently. + * + * We take a ShareLock on pg_partitioned_table to restrict addition of new + * partitioned table which may contain a foreign partition while publication is + * being created. + */ +void +check_foreign_tables(Oid schemaid, char *pubname) +{ + Relation classRel; + Relation partRel; + ScanKeyData key[3]; + int keycount = 0; + TableScanDesc scan; + HeapTuple tuple; + + classRel = table_open(RelationRelationId, AccessShareLock); + partRel = table_open(PartitionedRelationId, ShareLock); + + /* Get the root nodes of partitioned table */ + ScanKeyInit(&key[keycount++], + Anum_pg_class_relkind, + BTEqualStrategyNumber, F_CHAREQ, + CharGetDatum(RELKIND_PARTITIONED_TABLE)); + + ScanKeyInit(&key[keycount++], + Anum_pg_class_relispartition, + BTEqualStrategyNumber, F_BOOLEQ, + BoolGetDatum(false)); + + /* If schema id is provided check partitioned table in that schema */ + if (OidIsValid(schemaid)) + ScanKeyInit(&key[keycount++], + Anum_pg_class_relnamespace, + BTEqualStrategyNumber, F_OIDEQ, + schemaid); + + scan = table_beginscan_catalog(classRel, keycount, key); + while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) + { + Form_pg_class relForm = (Form_pg_class) GETSTRUCT(tuple); + Relation pubrel = table_open(relForm->oid, AccessShareLock); + + if (check_partrel_has_foreign_table(pubrel)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot set parameter \"%s\" to true for publication \"%s\"", + "publish_via_partition_root", pubname), + errdetail("partition table \"%s\" in publication contains a foreign partition", + get_rel_name(relForm->oid)))); + + table_close(pubrel, NoLock); + } + + table_endscan(scan); + table_close(classRel, AccessShareLock); + table_close(partRel, NoLock); +} diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c index c14e038d54f..98c1c82db61 100644 --- a/src/backend/commands/foreigncmds.c +++ b/src/backend/commands/foreigncmds.c @@ -21,6 +21,7 @@ #include "catalog/dependency.h" #include "catalog/indexing.h" #include "catalog/objectaccess.h" +#include "catalog/partition.h" #include "catalog/pg_foreign_data_wrapper.h" #include "catalog/pg_foreign_server.h" #include "catalog/pg_foreign_table.h" @@ -1423,6 +1424,55 @@ CreateForeignTable(CreateForeignTableStmt *stmt, Oid relid) ftrel = table_open(ForeignTableRelationId, RowExclusiveLock); + /* + * Check if it is a foreign partition and the partitioned table is not + * published or published with publish_via_partition_root option as false. + * See check_foreign_tables for details. + */ + if (stmt->base.partbound != NULL) + { + RangeVar *root = linitial_node(RangeVar, stmt->base.inhRelations); + Relation rootrel = table_openrv(root, AccessShareLock); + + if (RelationGetForm(rootrel)->relkind == RELKIND_PARTITIONED_TABLE) + { + Oid schemaid = RelationGetNamespace(rootrel); + List *puboids = GetRelationPublications(rootrel->rd_id); + List *ancestors; + + puboids = list_concat_unique_oid(puboids, GetAllTablesPublications()); + puboids = list_concat_unique_oid(puboids, GetSchemaPublications(schemaid)); + ancestors = get_partition_ancestors(rootrel->rd_id); + + foreach_oid(ancestor, ancestors) + { + puboids = list_concat_unique_oid(puboids, + GetRelationPublications(ancestor)); + schemaid = get_rel_namespace(ancestor); + puboids = list_concat_unique_oid(puboids, + GetSchemaPublications(schemaid)); + } + list_free(ancestors); + + foreach_oid(puboid, puboids) + { + Publication *pub = GetPublication(puboid); + + if (pub->pubviaroot) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot create table foreign partition \"%s\"", + get_rel_name(relid)), + errdetail("partition table \"%s\" is published with option publish_via_partition_root", + RelationGetRelationName(rootrel)))); + } + + list_free(puboids); + } + + table_close(rootrel, AccessShareLock); + } + /* * For now the owner cannot be specified on create. Use effective user ID. */ diff --git a/src/backend/commands/publicationcmds.c b/src/backend/commands/publicationcmds.c index 0b23d94c38e..b06960e8e22 100644 --- a/src/backend/commands/publicationcmds.c +++ b/src/backend/commands/publicationcmds.c @@ -915,6 +915,10 @@ CreatePublication(ParseState *pstate, CreatePublicationStmt *stmt) /* Associate objects with the publication. */ if (stmt->for_all_tables) { + /* Check if any partitioned table has foreign partition */ + if (publish_via_partition_root) + check_foreign_tables(InvalidOid, stmt->pubname); + /* Invalidate relcache so that publication info is rebuilt. */ CacheInvalidateRelcacheAll(); } @@ -1080,6 +1084,49 @@ AlterPublicationOptions(ParseState *pstate, AlterPublicationStmt *stmt, } } + /* + * If publish_via_partition_root is set to true, check if the publication + * already have any foreign partition. See check_foreign_tables for details. + */ + if (publish_via_partition_root_given && publish_via_partition_root) + { + List *schemaoids = NIL; + List *relids = NIL; + + char *pubname = stmt->pubname; + + if (pubform->puballtables) + check_foreign_tables(InvalidOid, pubname); + + schemaoids = GetPublicationSchemas(pubform->oid); + + foreach_oid(schemaoid, schemaoids) + check_foreign_tables(schemaoid, pubname); + + list_free(schemaoids); + + relids = GetPublicationRelations(pubform->oid, PUBLICATION_PART_ROOT); + + foreach_oid(relid, relids) + { + Relation pubrel = table_open(relid, AccessShareLock); + + if (check_partrel_has_foreign_table(pubrel)) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot set parameter \"%s\" to true for publication \"%s\"", + "publish_via_partition_root", pubname), + errdetail("partition table \"%s\" in publication contains a foreign partition", + get_rel_name(relid)))); + } + + table_close(pubrel, NoLock); + } + + list_free(relids); + } + /* Everything ok, form a new tuple. */ memset(values, 0, sizeof(values)); memset(nulls, false, sizeof(nulls)); diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 11fcb51a165..5c4007dbeee 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -19923,6 +19923,10 @@ QueuePartitionConstraintValidation(List **wqueue, Relation scanrel, * ALTER TABLE <name> ATTACH PARTITION <partition-name> FOR VALUES * * Return the address of the newly attached partition. + * + * If the would-be partition is a foreign table or a partitioned table with + * foreign partition, verify that the partitioned table is not in a publication + * with publish_via_partition_root=true. See check_foreign_tables for details. */ static ObjectAddress ATExecAttachPartition(List **wqueue, Relation rel, PartitionCmd *cmd, @@ -20067,6 +20071,61 @@ ATExecAttachPartition(List **wqueue, Relation rel, PartitionCmd *cmd, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot attach temporary relation of another session as partition"))); + /* + * Check if attachrel is a foreign table or a partitioned table with + * foreign partition and rel is not part of publication with option + * publish_via_partition_root as true. + */ + if (attachrel->rd_rel->relkind == RELKIND_FOREIGN_TABLE || + check_partrel_has_foreign_table(attachrel)) + { + Oid schemaid = RelationGetNamespace(rel); + List *puboids = GetRelationPublications(rel->rd_id); + List *ancestors; + char *relname = get_rel_name(rel->rd_id); + char *attachrelname = get_rel_name(attachrel->rd_id); + + puboids = list_concat_unique_oid(puboids, GetAllTablesPublications()); + puboids = list_concat_unique_oid(puboids, GetSchemaPublications(schemaid)); + ancestors = get_partition_ancestors(rel->rd_id); + + foreach_oid(ancestor, ancestors) + { + puboids = list_concat_unique_oid(puboids, + GetRelationPublications(ancestor)); + schemaid = get_rel_namespace(ancestor); + puboids = list_concat_unique_oid(puboids, + GetSchemaPublications(schemaid)); + } + + list_free(ancestors); + + foreach_oid(puboid, puboids) + { + Publication *pub = GetPublication(puboid); + + if (pub->pubviaroot) + { + if (attachrel->rd_rel->relkind == RELKIND_FOREIGN_TABLE) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot attach foreign table \"%s\" to partition table \"%s\"", + attachrelname, relname), + errdetail("partition table \"%s\" is published with option publish_via_partition_root", + relname))); + else + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot attach table \"%s\" with foreign partition to partition table \"%s\"", + attachrelname, relname), + errdetail("partition table \"%s\" is published with option publish_via_partition_root", + relname))); + } + } + + list_free(puboids); + } + /* * Check if attachrel has any identity columns or any columns that aren't * in the parent. diff --git a/src/include/catalog/pg_publication.h b/src/include/catalog/pg_publication.h index 48c7d1a8615..e2919da2541 100644 --- a/src/include/catalog/pg_publication.h +++ b/src/include/catalog/pg_publication.h @@ -19,6 +19,7 @@ #include "catalog/genbki.h" #include "catalog/objectaddress.h" +#include "catalog/pg_class.h" #include "catalog/pg_publication_d.h" /* IWYU pragma: export */ /* ---------------- @@ -191,4 +192,8 @@ extern Bitmapset *pub_collist_to_bitmapset(Bitmapset *columns, Datum pubcols, extern Bitmapset *pub_form_cols_map(Relation relation, PublishGencolsType include_gencols_type); +extern bool check_partrel_has_foreign_table(Relation rel); + +extern void check_foreign_tables(Oid schemaid, char *pubname); + #endif /* PG_PUBLICATION_H */ diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out index 4de96c04f9d..6a0c245bd60 100644 --- a/src/test/regress/expected/publication.out +++ b/src/test/regress/expected/publication.out @@ -1924,6 +1924,95 @@ DROP PUBLICATION pub1; DROP PUBLICATION pub2; DROP TABLE gencols; RESET client_min_messages; +-- ====================================================== +-- Test when foreign table is a partition of a partitioned table on which +-- publication is created +SET client_min_messages = 'ERROR'; +CREATE FOREIGN DATA WRAPPER test_fdw; +CREATE SERVER fdw_server FOREIGN DATA WRAPPER test_fdw; +CREATE SCHEMA sch3; +CREATE TABLE sch3.tmain(id int) PARTITION BY RANGE(id); +CREATE TABLE sch3.part1 PARTITION OF sch3.tmain FOR VALUES FROM (0) TO (5); +CREATE TABLE sch3.part2(id int) PARTITION BY RANGE(id); +CREATE FOREIGN TABLE sch3.part2_1 PARTITION OF sch3.part2 FOR VALUES FROM (5) TO (10) SERVER fdw_server; +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part2 FOR VALUES FROM (5) TO (10); +-- Can't create publications with publish_via_partition_root = true, if table +-- has a foreign partition +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain WITH (publish_via_partition_root); +ERROR: cannot set parameter "publish_via_partition_root" to true for publication "pub1" +DETAIL: partition table "tmain" in publication contains a foreign partition +CREATE PUBLICATION pub1 FOR TABLES IN SCHEMA sch3 WITH (publish_via_partition_root); +ERROR: cannot set parameter "publish_via_partition_root" to true for publication "pub1" +DETAIL: partition table "tmain" in publication contains a foreign partition +CREATE PUBLICATION pub1 FOR ALL TABLES WITH (publish_via_partition_root); +ERROR: cannot set parameter "publish_via_partition_root" to true for publication "pub1" +DETAIL: partition table "tmain" in publication contains a foreign partition +-- Test when a partitioned table with foreign table as a partition is attached +-- to partitioned table which is already published +ALTER TABLE sch3.tmain DETACH PARTITION sch3.part2; +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain WITH (publish_via_partition_root); +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part2 FOR VALUES FROM (5) TO (10); +ERROR: cannot attach table "part2" with foreign partition to partition table "tmain" +DETAIL: partition table "tmain" is published with option publish_via_partition_root +-- Can't create foreign partition of published table with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch3.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; +ERROR: cannot create table foreign partition "part3_1" +DETAIL: partition table "tmain" is published with option publish_via_partition_root +-- Can't attach foreign partition to published table with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_2(id int) SERVER fdw_server; +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); +ERROR: cannot attach foreign table "part3_2" to partition table "tmain" +DETAIL: partition table "tmain" is published with option publish_via_partition_root +CREATE SCHEMA sch4; +CREATE TABLE sch4.tmain(id int) PARTITION BY RANGE(id); +-- publication created with FOR TABLES IN SCHEMA +DROP PUBLICATION pub1; +CREATE PUBLICATION pub1 FOR TABLES IN SCHEMA sch4 WITH (publish_via_partition_root); +-- Can't create foreign partition of published table with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch4.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; +ERROR: cannot create table foreign partition "part3_1" +DETAIL: partition table "tmain" is published with option publish_via_partition_root +-- Can't attach foreign partition to published table with +-- publish_via_partition_root = true +ALTER TABLE sch4.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); +ERROR: cannot attach foreign table "part3_2" to partition table "tmain" +DETAIL: partition table "tmain" is published with option publish_via_partition_root +DROP PUBLICATION pub1; +-- Test with publish_via_partition_root = false +-- Foreign partitions are skipped by default +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain; +CREATE PUBLICATION pub2 FOR TABLES IN SCHEMA sch3; +CREATE PUBLICATION pub3 FOR ALL TABLES; +-- Create foreign partition of published table with +-- publish_via_partition_root = false +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch3.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; +-- Attach foreign partition to published table +-- publish_via_partition_root = false +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); +-- Check the published tables +SELECT pubname, tablename FROM pg_publication_tables WHERE schemaname in ('sch3', 'sch4') ORDER BY pubname, tablename; + pubname | tablename +---------+----------- + pub1 | part1 + pub2 | part1 + pub3 | part1 +(3 rows) + +-- Can't alter publish_via_partition_root to true, if publication already have +-- foreign partition +ALTER PUBLICATION pub1 SET (publish_via_partition_root); +ERROR: cannot set parameter "publish_via_partition_root" to true for publication "pub1" +DETAIL: partition table "tmain" in publication contains a foreign partition +DROP PUBLICATION pub1; +DROP PUBLICATION pub2; +DROP PUBLICATION pub3; +DROP SCHEMA sch3 CASCADE; +DROP SCHEMA sch4 CASCADE; +DROP SERVER fdw_server; +DROP FOREIGN DATA WRAPPER test_fdw; RESET SESSION AUTHORIZATION; DROP ROLE regress_publication_user, regress_publication_user2; DROP ROLE regress_publication_user_dummy; diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql index 68001de4000..49c9d98b668 100644 --- a/src/test/regress/sql/publication.sql +++ b/src/test/regress/sql/publication.sql @@ -1223,6 +1223,87 @@ DROP PUBLICATION pub2; DROP TABLE gencols; RESET client_min_messages; +-- ====================================================== + +-- Test when foreign table is a partition of a partitioned table on which +-- publication is created +SET client_min_messages = 'ERROR'; +CREATE FOREIGN DATA WRAPPER test_fdw; +CREATE SERVER fdw_server FOREIGN DATA WRAPPER test_fdw; + +CREATE SCHEMA sch3; +CREATE TABLE sch3.tmain(id int) PARTITION BY RANGE(id); +CREATE TABLE sch3.part1 PARTITION OF sch3.tmain FOR VALUES FROM (0) TO (5); +CREATE TABLE sch3.part2(id int) PARTITION BY RANGE(id); +CREATE FOREIGN TABLE sch3.part2_1 PARTITION OF sch3.part2 FOR VALUES FROM (5) TO (10) SERVER fdw_server; +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part2 FOR VALUES FROM (5) TO (10); + +-- Can't create publications with publish_via_partition_root = true, if table +-- has a foreign partition +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain WITH (publish_via_partition_root); +CREATE PUBLICATION pub1 FOR TABLES IN SCHEMA sch3 WITH (publish_via_partition_root); +CREATE PUBLICATION pub1 FOR ALL TABLES WITH (publish_via_partition_root); + +-- Test when a partitioned table with foreign table as a partition is attached +-- to partitioned table which is already published +ALTER TABLE sch3.tmain DETACH PARTITION sch3.part2; +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain WITH (publish_via_partition_root); +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part2 FOR VALUES FROM (5) TO (10); + +-- Can't create foreign partition of published table with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch3.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; + +-- Can't attach foreign partition to published table with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_2(id int) SERVER fdw_server; +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); + +CREATE SCHEMA sch4; +CREATE TABLE sch4.tmain(id int) PARTITION BY RANGE(id); + +-- publication created with FOR TABLES IN SCHEMA +DROP PUBLICATION pub1; +CREATE PUBLICATION pub1 FOR TABLES IN SCHEMA sch4 WITH (publish_via_partition_root); + +-- Can't create foreign partition of published table with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch4.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; + +-- Can't attach foreign partition to published table with +-- publish_via_partition_root = true +ALTER TABLE sch4.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); +DROP PUBLICATION pub1; + +-- Test with publish_via_partition_root = false +-- Foreign partitions are skipped by default +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain; +CREATE PUBLICATION pub2 FOR TABLES IN SCHEMA sch3; +CREATE PUBLICATION pub3 FOR ALL TABLES; + +-- Create foreign partition of published table with +-- publish_via_partition_root = false +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch3.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; + +-- Attach foreign partition to published table +-- publish_via_partition_root = false +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); + +-- Check the published tables +SELECT pubname, tablename FROM pg_publication_tables WHERE schemaname in ('sch3', 'sch4') ORDER BY pubname, tablename; + +-- Can't alter publish_via_partition_root to true, if publication already have +-- foreign partition +ALTER PUBLICATION pub1 SET (publish_via_partition_root); + +DROP PUBLICATION pub1; +DROP PUBLICATION pub2; +DROP PUBLICATION pub3; +DROP SCHEMA sch3 CASCADE; +DROP SCHEMA sch4 CASCADE; +DROP SERVER fdw_server; +DROP FOREIGN DATA WRAPPER test_fdw; + RESET SESSION AUTHORIZATION; DROP ROLE regress_publication_user, regress_publication_user2; DROP ROLE regress_publication_user_dummy; -- 2.34.1 ^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: Restrict publishing of partitioned table with a foreign table as partition @ 2025-04-28 12:48 ` Shlok Kyal <[email protected]> 2025-04-28 14:27 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> 1 sibling, 1 reply; 19+ messages in thread From: Shlok Kyal @ 2025-04-28 12:48 UTC (permalink / raw) To: Álvaro Herrera <[email protected]>; +Cc: vignesh C <[email protected]>; Sergey Tatarintsev <[email protected]>; [email protected] On Mon, 7 Apr 2025 at 18:09, Álvaro Herrera <[email protected]> wrote: > > Here's the additional changes I made here before giving up on this. > I think it needs some additional rethinking, not going to happen for 18. > Hi Alvaro, Thanks for reviewing the patch. The changes shared by you in [1], look good to me and I have added it to the latest patch. I have also included the changes shared by you in [2]. I have also addressed the issue reported by Sergey in [3]. Additionally, I made some changes to the comments, commit message and documentation to reflect the changes in the function name and to reduce the usage of 'foreign partition' term. I saw some comments by you in the patches: 1. + /* + * Keep lock till end of transaction: must prevent this table from + * being attached a foreign table until we're done. XXX does this + * prevent addition of a partition in a partitioned child? + */ This is same as the issue reported by Sergey in [3]. I have addressed the issue in the latest patch. So, I have modified this comment. 2. + * We also take a ShareLock on pg_partitioned_table to restrict addition + * of new partitioned table which may contain a foreign partition while + * publication is being created. XXX this is quite weird actually. This change was added to resolve the concurrency issue shared by Vignesh in [4]. I tried with different locks and found that lock with severity >= ShareLock was needed to avoid the concurrency issue. Initially I added ShareLock to pg_class, but to reduce the scope I added it to pg_partitioned_table instead. I cannot think of an alternate approach. Do you have any suggestions for this? [1]: https://www.postgresql.org/message-id/202504062027.tqmabk2h353o%40alvherre.pgsql [2]: https://www.postgresql.org/message-id/202504071239.kuj6m5a5wqxg%40alvherre.pgsql [3]: https://www.postgresql.org/message-id/c64352fa-9a30-4e0a-853a-a6b5b6d07f4e%40postgrespro.ru [4]: https://www.postgresql.org/message-id/CALDaNm2%2BeL22Sbvj74uS37xvt%3DhaQWcOwP15QnDuVeYsjHiffw%40mail... Thanks and Regards, Shlok Kyal Attachments: [application/octet-stream] v13-0001-Restrict-publishing-of-partitioned-table-with-fo.patch (35.9K, ../../CANhcyEW0QMiJXMqpPFRHni-q0Rm4R0hpZ0LdaqA=F3wvDUU6sQ@mail.gmail.com/2-v13-0001-Restrict-publishing-of-partitioned-table-with-fo.patch) download | inline diff: From 534eee027b02e5cda9b502f9f624a5fb426d7f84 Mon Sep 17 00:00:00 2001 From: Shlok Kyal <[email protected]> Date: Fri, 28 Mar 2025 11:15:09 +0530 Subject: [PATCH v13] Restrict publishing of partitioned table with foreign table as its partition Logical replication of foreign table is not supported and we throw an error in this case. But when we create a publication on a partitioned table that has a foreign table as its partition, the initial sync of such table is successful and we should avoid such cases. Current Behaviour in HEAD, when publication is created: 1. with publish_via_partition_root = true The root table is published, and initial data from partitions that are foreign tables is replicated. 2. with publish_via_partition_root = false and FOR ALL TABLES All leaf tables except partitions that are foreign tables are published. 3. with publish_via_partition_root = false and FOR TABLE/ FOR TABLES IN SCHEMA All leaf tables are published, including initial data from partitions that are foreign tables. With this patch we have following behaviour: 1. with publish_via_partition_root = true We throw an error when we try to publish a partition that is a foreign table. Error is thrown when we try to create a publication on (or add to existing publication) a partitioned table that has foreign tables as its partitions, when try to create a partition that is a foreign table and when we try to attach foreign table (or a partitioned table with foreign tables as partitions) to existing published tables. 2. with publish_via_partition_root = false We skip publishing partitions that are foreign tables. This is done by avoid adding such partitions in pg_subscription_rel catalog table. We have introduced two functions 'RelationHasForeignPartition' and 'publication_check_foreign_parts'. In 'RelationHasForeignPartition' we go through the child nodes of a partition and check if it has a foreign table. While doing so, we take an AccessShareLock on each partition table to prevent concurrent creation of a foreign table as a partition. In 'publication_check_foreign_parts' if schema id is provided we check for each partitioned table in that schema if it has a foreign table as its partition, or if schema id is not provided we check for each partitioned table in the database if it has a partition that's a foreign table. While doing so we take a ShareLock on pg_partitioned_table so no partition table is created concurrently after this check. --- doc/src/sgml/logical-replication.sgml | 10 +- doc/src/sgml/ref/create_publication.sgml | 18 +- src/backend/catalog/pg_publication.c | 221 ++++++++++++++++++++-- src/backend/commands/publicationcmds.c | 47 +++++ src/backend/commands/tablecmds.c | 112 +++++++++++ src/backend/partitioning/partdesc.c | 32 ++++ src/include/catalog/pg_publication.h | 3 + src/include/partitioning/partdesc.h | 1 + src/test/regress/expected/publication.out | 99 ++++++++++ src/test/regress/sql/publication.sql | 90 +++++++++ 10 files changed, 611 insertions(+), 22 deletions(-) diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index f288c049a5c..57236a452e9 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -2154,10 +2154,18 @@ CONTEXT: processing remote data for replication origin "pg_16395" during "INSER <listitem> <para> - Replication is only supported by tables, including partitioned tables. + Replication is only supported for tables, including partitioned tables. Attempts to replicate other types of relations, such as views, materialized views, or foreign tables, will result in an error. </para> + <para> + Replication is not supported for foreign tables. When used as partitions + of partitioned tables, publishing of the partitioned table is only allowed + if the <literal>publish_via_partition_root</literal> is set to + <literal>false</literal>. In this mode, changes to a partition that is a + foreign table are ignored for the purposes of replication, and data + contained in them is not included during initial synchronization. + </para> </listitem> <listitem> diff --git a/doc/src/sgml/ref/create_publication.sgml b/doc/src/sgml/ref/create_publication.sgml index 73f0c8d89fb..ad2c3b69d6b 100644 --- a/doc/src/sgml/ref/create_publication.sgml +++ b/doc/src/sgml/ref/create_publication.sgml @@ -249,13 +249,23 @@ CREATE PUBLICATION <replaceable class="parameter">name</replaceable> </para> <para> - This parameter also affects how row filters and column lists are - chosen for partitions; see below for details. + If this parameter is enabled, <literal>TRUNCATE</literal> + operations performed directly on partitions are not replicated. </para> <para> - If this is enabled, <literal>TRUNCATE</literal> operations performed - directly on partitions are not replicated. + If this parameter is enabled, foreign tables and partitioned tables + containing partitions that are foreign tables may not be + added to the publication. Conversely, foreign tables may not be + attached to a partitioned table that is included in a publication + with this parameter enabled. Lastly, this parameter may not be + changed on publications that include partitioned tables with foreign + tables as partitions. + </para> + + <para> + This parameter also affects how row filters and column lists are + chosen for partitions; see below for details. </para> </listitem> </varlistentry> diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c index d6f94db5d99..73a9bf22470 100644 --- a/src/backend/catalog/pg_publication.c +++ b/src/backend/catalog/pg_publication.c @@ -26,12 +26,15 @@ #include "catalog/partition.h" #include "catalog/pg_inherits.h" #include "catalog/pg_namespace.h" +#include "catalog/pg_partitioned_table.h" #include "catalog/pg_publication.h" #include "catalog/pg_publication_namespace.h" #include "catalog/pg_publication_rel.h" #include "catalog/pg_type.h" #include "commands/publicationcmds.h" #include "funcapi.h" +#include "partitioning/partdesc.h" +#include "storage/lmgr.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/catcache.h" @@ -53,7 +56,7 @@ typedef struct * error if not. */ static void -check_publication_add_relation(Relation targetrel) +check_publication_add_relation(Publication *pub, Relation targetrel) { /* Must be a regular or partitioned table */ if (RelationGetForm(targetrel)->relkind != RELKIND_RELATION && @@ -64,6 +67,21 @@ check_publication_add_relation(Relation targetrel) RelationGetRelationName(targetrel)), errdetail_relkind_not_supported(RelationGetForm(targetrel)->relkind))); + /* + * publish_via_root_partition cannot be true if it is a partitioned table + * and has any partition that's a foreign table. See + * publication_check_foreign_parts for details. + */ + if (pub->pubviaroot && + targetrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE && + RelationHasForeignPartition(targetrel)) + ereport(ERROR, + errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot add partitioned table \"%s\" to publication \"%s\", which has \"%s\" set to \"%s\"", + RelationGetRelationName(targetrel), pub->name, + "publish_via_partition_root", "true"), + errdetail("The table contains a partition that's a foreign table.")); + /* Can't be system table */ if (IsCatalogRelation(targetrel)) ereport(ERROR, @@ -111,6 +129,172 @@ check_publication_add_schema(Oid schemaid) errdetail("Temporary schemas cannot be replicated."))); } +/* + * Returns true if the ancestor is in the list of relations + * Otherwise, returns false. + */ +static bool +is_ancestor_member_relids(Oid ancestor, List *relids) +{ + foreach_oid(relid, relids) + { + if (relid == ancestor) + return true; + } + + return false; +} + +/* + * Filter out the partitions whose parent tables are also present in the list + * of relations. + */ +static void +filter_partition_rels(List *relids) +{ + foreach_oid(relid, relids) + { + bool skip = false; + List *ancestors = NIL; + ListCell *lc2; + + if (get_rel_relispartition(relid)) + ancestors = get_partition_ancestors(relid); + + foreach_oid(ancestor, ancestors) + { + if (is_ancestor_member_relids(ancestor, relids)) + { + skip = true; + break; + } + } + + if (skip) + relids = list_delete_oid(relids, relid); + } +} + +/* + * publication_check_foreign_parts + * Helper function to ensure we don't indirectly publish foreign tables + * + * DML data changes are not published for data in foreign tables, + * and yet the tablesync worker is not smart enough to omit data from + * foreign tables when they are partitions of partitioned tables. To + * avoid the inconsistencies that would result, we disallow foreign + * tables from being published generally. However, it's possible for + * partitioned tables to have foreign tables as partitions, and we would + * like to allow publishing those partitioned tables so that the other + * partitions are replicated. + * + * This function is in charge of detecting if a partitioned table has a + * foreign table as a partition -- either in the whole database (useful + * for FOR ALL TABLES publications) or in a particular schema (useful + * for FOR TABLES IN SCHEMA publications). This function must be called + * only for publications with publish_via_partition_root=true. + * + * When publish_via_partition_root is false, each partition published for + * replication is listed individually in pg_subscription_rel, and we + * don't add partitions that are foreign tables, so this check is not + * needed. + * + * If a valid schemaid is provided, check if that schema has any + * partitioned table with a foreign table as partition. + * + * If no valid schemaid is provided, check all partitioned tables. + * + * We take a lock on partition tables so no new foreign table are added + * concurrently as a partition. + * + * We also take a ShareLock on pg_partitioned_table to restrict addition + * of new partitioned table which may include a foreign table as partition + * while publication is being created. XXX this is quite weird actually. + */ +void +publication_check_foreign_parts(Oid schemaid, char *pubname) +{ + Relation classRel; + ScanKeyData key[3]; + int keycount = 0; + TableScanDesc scan; + HeapTuple tuple; + List *relids = NIL; + + /* + * Take lock on pg_partitioned_rel. This prevents new publications from + * being created. + */ + LockRelationOid(PartitionedRelationId, ShareLock); + + classRel = table_open(RelationRelationId, AccessShareLock); + + /* Get the root nodes of partitioned table */ + ScanKeyInit(&key[keycount++], + Anum_pg_class_relkind, + BTEqualStrategyNumber, F_CHAREQ, + CharGetDatum(RELKIND_PARTITIONED_TABLE)); + + /* If schema id is provided check partitioned table in that schema */ + if (OidIsValid(schemaid)) + ScanKeyInit(&key[keycount++], + Anum_pg_class_relnamespace, + BTEqualStrategyNumber, F_OIDEQ, + schemaid); + + /* + * If schema id is not provided, take relations for which relispartition + * is false. This will give only the root partitioned tables. We donot + * include this for the case when schema id is specified because there can + * be cases when root partitioned tables are not part of schema and one of + * the child partition can still have a foreign table as its partition. + */ + else + ScanKeyInit(&key[keycount++], + Anum_pg_class_relispartition, + BTEqualStrategyNumber, F_BOOLEQ, + BoolGetDatum(false)); + + scan = table_beginscan_catalog(classRel, keycount, key); + while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) + { + Form_pg_class relForm = (Form_pg_class) GETSTRUCT(tuple); + + relids = lappend_oid(relids, relForm->oid); + } + + /* + * If schema id is provided filter partitions list to have only topmost + * partitioned tables in that schema, to avoid repeated check. + */ + if (OidIsValid(schemaid)) + filter_partition_rels(relids); + + foreach_oid(relid, relids) + { + Relation pubrel = table_open(relid, AccessShareLock); + + if (pubrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE && + RelationHasForeignPartition(pubrel)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot set parameter \"%s\" to \"%s\" for publication \"%s\"", + "publish_via_partition_root", "true", pubname), + errtable(pubrel), + errdetail("Published partitioned table \"%s\" contains a partition that is a foreign table.", + get_rel_name(relid)))); + + /* + * Keep lock till end of transaction: must prevent this table from + * being attached a foreign table until we're done. + */ + table_close(pubrel, NoLock); + } + + table_endscan(scan); + table_close(classRel, AccessShareLock); +} + /* * Returns if relation represented by oid and Form_pg_class entry * is publishable. @@ -304,7 +488,7 @@ check_and_fetch_column_list(Publication *pub, Oid relid, MemoryContext mcxt, /* * Gets the relations based on the publication partition option for a specified - * relation. + * relation. Foreign tables are not included. */ List * GetPubPartitionOptionRelations(List *result, PublicationPartOpt pub_partopt, @@ -313,25 +497,21 @@ GetPubPartitionOptionRelations(List *result, PublicationPartOpt pub_partopt, if (get_rel_relkind(relid) == RELKIND_PARTITIONED_TABLE && pub_partopt != PUBLICATION_PART_ROOT) { - List *all_parts = find_all_inheritors(relid, NoLock, - NULL); + List *all_parts = find_all_inheritors(relid, NoLock, NULL); - if (pub_partopt == PUBLICATION_PART_ALL) - result = list_concat(result, all_parts); - else if (pub_partopt == PUBLICATION_PART_LEAF) + foreach_oid(partOid, all_parts) { - ListCell *lc; + char relkind = get_rel_relkind(partOid); - foreach(lc, all_parts) - { - Oid partOid = lfirst_oid(lc); + if (relkind == RELKIND_FOREIGN_TABLE) + continue; - if (get_rel_relkind(partOid) != RELKIND_PARTITIONED_TABLE) - result = lappend_oid(result, partOid); - } + if (pub_partopt == PUBLICATION_PART_LEAF && + relkind == RELKIND_PARTITIONED_TABLE) + continue; + + result = lappend_oid(result, partOid); } - else - Assert(false); } else result = lappend_oid(result, relid); @@ -463,7 +643,7 @@ publication_add_relation(Oid pubid, PublicationRelInfo *pri, RelationGetRelationName(targetrel), pub->name))); } - check_publication_add_relation(targetrel); + check_publication_add_relation(pub, targetrel); /* Validate and translate column names into a Bitmapset of attnums. */ attnums = pub_collist_validate(pri->relation, pri->columns); @@ -703,6 +883,13 @@ publication_add_schema(Oid pubid, Oid schemaid, bool if_not_exists) check_publication_add_schema(schemaid); + /* + * If publish_via_partition_root is true, check if schema has any foreign + * partition + */ + if (pub->pubviaroot) + publication_check_foreign_parts(schemaid, pub->name); + /* Form a tuple */ memset(values, 0, sizeof(values)); memset(nulls, false, sizeof(nulls)); diff --git a/src/backend/commands/publicationcmds.c b/src/backend/commands/publicationcmds.c index 0b23d94c38e..a2411fbebda 100644 --- a/src/backend/commands/publicationcmds.c +++ b/src/backend/commands/publicationcmds.c @@ -35,6 +35,7 @@ #include "commands/publicationcmds.h" #include "miscadmin.h" #include "nodes/nodeFuncs.h" +#include "partitioning/partdesc.h" #include "parser/parse_clause.h" #include "parser/parse_collate.h" #include "parser/parse_relation.h" @@ -915,6 +916,12 @@ CreatePublication(ParseState *pstate, CreatePublicationStmt *stmt) /* Associate objects with the publication. */ if (stmt->for_all_tables) { + /* + * Check if any partitioned table has a foreign table as its partition + */ + if (publish_via_partition_root) + publication_check_foreign_parts(InvalidOid, stmt->pubname); + /* Invalidate relcache so that publication info is rebuilt. */ CacheInvalidateRelcacheAll(); } @@ -1080,6 +1087,46 @@ AlterPublicationOptions(ParseState *pstate, AlterPublicationStmt *stmt, } } + /* + * If publish_via_partition_root is set to true, check if the publication + * has any partition that's a foreign table. See + * publication_check_foreign_parts for details. + */ + if (publish_via_partition_root_given && publish_via_partition_root) + { + char *pubname = stmt->pubname; + List *schemaoids; + List *relids; + + if (pubform->puballtables) + publication_check_foreign_parts(InvalidOid, pubname); + + schemaoids = GetPublicationSchemas(pubform->oid); + foreach_oid(schemaoid, schemaoids) + publication_check_foreign_parts(schemaoid, pubname); + + relids = GetPublicationRelations(pubform->oid, PUBLICATION_PART_ROOT); + foreach_oid(relid, relids) + { + Relation pubrel; + + if (get_rel_relkind(relid) != RELKIND_PARTITIONED_TABLE) + continue; + + pubrel = table_open(relid, AccessShareLock); + + if (RelationHasForeignPartition(pubrel)) + ereport(ERROR, + errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot set parameter \"%s\" to \"%s\" for publication \"%s\"", + "publish_via_partition_root", "true", pubname), + errdetail("Published partitioned table \"%s\" contains a partition that is a foreign table.", + RelationGetRelationName(pubrel))); + + table_close(pubrel, NoLock); + } + } + /* Everything ok, form a new tuple. */ memset(values, 0, sizeof(values)); memset(nulls, false, sizeof(nulls)); diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 2705cf11330..3587e9960d8 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -1133,6 +1133,53 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, errmsg("\"%s\" is not partitioned", RelationGetRelationName(parent)))); + /* + * If we're creating a partition that's a foreign table, verify that + * the parent table is not in a publication with + * publish_via_partition_root enabled. For details, see + * publication_check_foreign_parts. + */ + if (rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE) + { + Oid schemaid; + List *puboids; + List *ancestors; + + /* Start with publications of all tables */ + puboids = GetAllTablesPublications(); + + /* capture all publications that include this relation directly */ + puboids = GetRelationPublications(parent->rd_id); + schemaid = RelationGetNamespace(parent); + puboids = list_concat(puboids, GetSchemaPublications(schemaid)); + + /* and do the same for its ancestors, if any */ + ancestors = get_partition_ancestors(parent->rd_id); + foreach_oid(ancestor, ancestors) + { + puboids = list_concat(puboids, GetRelationPublications(ancestor)); + schemaid = get_rel_namespace(ancestor); + puboids = list_concat(puboids, GetSchemaPublications(schemaid)); + } + + /* Check the publish_via_partition_root bit for each of those */ + list_sort(puboids, list_oid_cmp); + list_deduplicate_oid(puboids); + foreach_oid(puboid, puboids) + { + Publication *pub = GetPublication(puboid); + + if (pub->pubviaroot) + ereport(ERROR, + errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot create foreign table \"%s\" as a partition of \"%s\"", + RelationGetRelationName(rel), RelationGetRelationName(parent)), + errdetail("Partitioned table \"%s\" is published with option \"%s\" in publication \"%s\".", + RelationGetRelationName(parent), + "publish_via_partition_root", pub->name)); + } + } + /* * The partition constraint of the default partition depends on the * partition bounds of every other partition. It is possible that @@ -20304,6 +20351,71 @@ ATExecAttachPartition(List **wqueue, Relation rel, PartitionCmd *cmd, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot attach temporary relation of another session as partition"))); + /* + * If the relation to attach is a foreign table, or a partitioned table + * that contains a foreign table as partition, then verify that the parent + * table is not in a publication with publish_via_partition_root enabled. + * See publication_check_foreign_parts. + */ + if (attachrel->rd_rel->relkind == RELKIND_FOREIGN_TABLE || + (attachrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE && + RelationHasForeignPartition(attachrel))) + { + Oid schemaid; + List *puboids; + List *ancestors; + + /* Start with publications of all tables */ + puboids = GetAllTablesPublications(); + + /* capture all publications that include this relation directly */ + puboids = list_concat(puboids, GetRelationPublications(rel->rd_id)); + schemaid = RelationGetNamespace(rel); + puboids = list_concat(puboids, GetSchemaPublications(schemaid)); + + /* and do the same for its ancestors, if any */ + ancestors = get_partition_ancestors(rel->rd_id); + foreach_oid(ancestor, ancestors) + { + puboids = list_concat(puboids, GetRelationPublications(ancestor)); + schemaid = get_rel_namespace(ancestor); + puboids = list_concat(puboids, GetSchemaPublications(schemaid)); + } + + /* Now check the publish_via_partition_root bit for each of those */ + list_sort(puboids, list_oid_cmp); + list_deduplicate_oid(puboids); + foreach_oid(puboid, puboids) + { + Publication *pub; + + pub = GetPublication(puboid); + if (pub->pubviaroot) + { + if (attachrel->rd_rel->relkind == RELKIND_FOREIGN_TABLE) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot attach foreign table \"%s\" to partition table \"%s\"", + RelationGetRelationName(attachrel), + RelationGetRelationName(rel)), + errdetail("Partitioned table \"%s\" is published with option \"%s\" in publication \"%s\".", + RelationGetRelationName(rel), + "publish_via_partition_root", + pub->name))); + else + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot attach table \"%s\" with a partition that's a foreign table to partition table \"%s\"", + RelationGetRelationName(attachrel), + RelationGetRelationName(rel)), + errdetail("Partitioned table \"%s\" is published with option \"%s\" in publication \"%s\".", + RelationGetRelationName(rel), + "publish_via_partition_root", + pub->name))); + } + } + } + /* * Check if attachrel has any identity columns or any columns that aren't * in the parent. diff --git a/src/backend/partitioning/partdesc.c b/src/backend/partitioning/partdesc.c index 328b4d450e4..b53139bafdd 100644 --- a/src/backend/partitioning/partdesc.c +++ b/src/backend/partitioning/partdesc.c @@ -506,3 +506,35 @@ get_default_oid_from_partdesc(PartitionDesc partdesc) return InvalidOid; } + +/* + * Return true if the given partitioned table ultimately contains a + * partition that is a foreign table, false otherwise. + */ +bool +RelationHasForeignPartition(Relation rel) +{ + PartitionDesc pd = RelationGetPartitionDesc(rel, true); + + for (int i = 0; i < pd->nparts; i++) + { + if (pd->is_leaf[i]) + { + if (get_rel_relkind(pd->oids[i]) == RELKIND_FOREIGN_TABLE) + return true; + } + else + { + Relation part; + bool ret; + + part = table_open(pd->oids[i], AccessShareLock); + ret = RelationHasForeignPartition(part); + table_close(part, NoLock); + if (ret) + return true; + } + } + + return false; +} diff --git a/src/include/catalog/pg_publication.h b/src/include/catalog/pg_publication.h index 48c7d1a8615..71ad5a6f846 100644 --- a/src/include/catalog/pg_publication.h +++ b/src/include/catalog/pg_publication.h @@ -19,6 +19,7 @@ #include "catalog/genbki.h" #include "catalog/objectaddress.h" +#include "catalog/pg_class.h" #include "catalog/pg_publication_d.h" /* IWYU pragma: export */ /* ---------------- @@ -178,6 +179,8 @@ extern Oid GetTopMostAncestorInPublication(Oid puboid, List *ancestors, extern bool is_publishable_relation(Relation rel); extern bool is_schema_publication(Oid pubid); +extern void publication_check_foreign_parts(Oid schemaid, char *pubname); + extern bool check_and_fetch_column_list(Publication *pub, Oid relid, MemoryContext mcxt, Bitmapset **cols); extern ObjectAddress publication_add_relation(Oid pubid, PublicationRelInfo *pri, diff --git a/src/include/partitioning/partdesc.h b/src/include/partitioning/partdesc.h index 34533f7004c..5fbafdc06f9 100644 --- a/src/include/partitioning/partdesc.h +++ b/src/include/partitioning/partdesc.h @@ -71,5 +71,6 @@ extern PartitionDesc PartitionDirectoryLookup(PartitionDirectory, Relation); extern void DestroyPartitionDirectory(PartitionDirectory pdir); extern Oid get_default_oid_from_partdesc(PartitionDesc partdesc); +extern bool RelationHasForeignPartition(Relation rel); #endif /* PARTDESC_H */ diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out index 4de96c04f9d..f57addab23d 100644 --- a/src/test/regress/expected/publication.out +++ b/src/test/regress/expected/publication.out @@ -1924,6 +1924,105 @@ DROP PUBLICATION pub1; DROP PUBLICATION pub2; DROP TABLE gencols; RESET client_min_messages; +-- ====================================================== +-- Test when foreign table is a partition of a partitioned table on which +-- publication is created +SET client_min_messages = 'ERROR'; +CREATE FOREIGN DATA WRAPPER test_fdw; +CREATE SERVER fdw_server FOREIGN DATA WRAPPER test_fdw; +CREATE SCHEMA sch3; +CREATE TABLE sch3.tmain(id int) PARTITION BY RANGE(id); +CREATE TABLE sch3.part1 PARTITION OF sch3.tmain FOR VALUES FROM (0) TO (5); +CREATE TABLE sch3.part2(id int) PARTITION BY RANGE(id); +CREATE FOREIGN TABLE sch3.part2_1 PARTITION OF sch3.part2 FOR VALUES FROM (5) TO (10) SERVER fdw_server; +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part2 FOR VALUES FROM (5) TO (10); +-- Can't create publications with publish_via_partition_root = true, if table +-- has a partition that's a foreign table +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain WITH (publish_via_partition_root); +ERROR: cannot add partitioned table "tmain" to publication "pub1", which has "publish_via_partition_root" set to "true" +DETAIL: The table contains a partition that's a foreign table. +CREATE PUBLICATION pub1 FOR TABLES IN SCHEMA sch3 WITH (publish_via_partition_root); +ERROR: cannot set parameter "publish_via_partition_root" to "true" for publication "pub1" +DETAIL: Published partitioned table "tmain" contains a partition that is a foreign table. +CREATE PUBLICATION pub1 FOR ALL TABLES WITH (publish_via_partition_root); +ERROR: cannot set parameter "publish_via_partition_root" to "true" for publication "pub1" +DETAIL: Published partitioned table "tmain" contains a partition that is a foreign table. +-- Test when a partitioned table with foreign table as a partition is attached +-- to partitioned table which is already published +ALTER TABLE sch3.tmain DETACH PARTITION sch3.part2; +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain WITH (publish_via_partition_root); +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part2 FOR VALUES FROM (5) TO (10); +ERROR: cannot attach table "part2" with a partition that's a foreign table to partition table "tmain" +DETAIL: Partitioned table "tmain" is published with option "publish_via_partition_root" in publication "pub1". +-- Can't create a foreign table that is partition of table published with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch3.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; +ERROR: cannot create foreign table "part3_1" as a partition of "tmain" +DETAIL: Partitioned table "tmain" is published with option "publish_via_partition_root" in publication "pub1". +-- Can't attach a foreign table as partition to table published with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_2(id int) SERVER fdw_server; +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); +ERROR: cannot attach foreign table "part3_2" to partition table "tmain" +DETAIL: Partitioned table "tmain" is published with option "publish_via_partition_root" in publication "pub1". +CREATE SCHEMA sch4; +CREATE TABLE sch4.tmain(id int) PARTITION BY RANGE(id); +-- publication created with FOR TABLES IN SCHEMA +DROP PUBLICATION pub1; +CREATE PUBLICATION pub1 FOR TABLES IN SCHEMA sch4 WITH (publish_via_partition_root); +-- Can't create a foreign table that is partition of table published with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch4.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; +ERROR: cannot create foreign table "part3_1" as a partition of "tmain" +DETAIL: Partitioned table "tmain" is published with option "publish_via_partition_root" in publication "pub1". +-- Can't attach a foreign table as partition to table published with +-- publish_via_partition_root = true +ALTER TABLE sch4.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); +ERROR: cannot attach foreign table "part3_2" to partition table "tmain" +DETAIL: Partitioned table "tmain" is published with option "publish_via_partition_root" in publication "pub1". +DROP PUBLICATION pub1; +-- Can't create publication with publish_via_partition_root = true on +-- partitioned table(which is not root) with a partition that's a foreign table +-- on other schema +CREATE SCHEMA sch5; +CREATE TABLE sch5.part1 PARTITION OF sch4.tmain FOR VALUES FROM (0) TO (10) PARTITION BY RANGE(id); +CREATE FOREIGN TABLE sch4.part1_1 PARTITION OF sch5.part1 FOR VALUES FROM (0) TO (5) SERVER fdw_server; +CREATE PUBLICATION pub1 FOR TABLES IN SCHEMA sch5 WITH (publish_via_partition_root); +ERROR: cannot set parameter "publish_via_partition_root" to "true" for publication "pub1" +DETAIL: Published partitioned table "part1" contains a partition that is a foreign table. +-- Test with publish_via_partition_root = false +-- Partition that are foreign tables are skipped by default +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain; +CREATE PUBLICATION pub2 FOR TABLES IN SCHEMA sch3; +CREATE PUBLICATION pub3 FOR ALL TABLES; +-- Create a partition that's a foreign table of published table with +-- publish_via_partition_root = false +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch3.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; +-- Attach partition that's a foreign table to published table +-- publish_via_partition_root = false +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); +-- Check the published tables +SELECT pubname, tablename FROM pg_publication_tables WHERE schemaname in ('sch3', 'sch4') ORDER BY pubname, tablename; + pubname | tablename +---------+----------- + pub1 | part1 + pub2 | part1 + pub3 | part1 +(3 rows) + +-- Can't alter publish_via_partition_root to true, if publication already have +-- a partition that's a foreign table +ALTER PUBLICATION pub1 SET (publish_via_partition_root); +ERROR: cannot set parameter "publish_via_partition_root" to "true" for publication "pub1" +DETAIL: Published partitioned table "tmain" contains a partition that is a foreign table. +DROP PUBLICATION pub1; +DROP PUBLICATION pub2; +DROP PUBLICATION pub3; +DROP SCHEMA sch3 CASCADE; +DROP SCHEMA sch4 CASCADE; +DROP SCHEMA sch5 CASCADE; +DROP SERVER fdw_server; +DROP FOREIGN DATA WRAPPER test_fdw; RESET SESSION AUTHORIZATION; DROP ROLE regress_publication_user, regress_publication_user2; DROP ROLE regress_publication_user_dummy; diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql index 68001de4000..55c0dff3601 100644 --- a/src/test/regress/sql/publication.sql +++ b/src/test/regress/sql/publication.sql @@ -1223,6 +1223,96 @@ DROP PUBLICATION pub2; DROP TABLE gencols; RESET client_min_messages; +-- ====================================================== + +-- Test when foreign table is a partition of a partitioned table on which +-- publication is created +SET client_min_messages = 'ERROR'; +CREATE FOREIGN DATA WRAPPER test_fdw; +CREATE SERVER fdw_server FOREIGN DATA WRAPPER test_fdw; + +CREATE SCHEMA sch3; +CREATE TABLE sch3.tmain(id int) PARTITION BY RANGE(id); +CREATE TABLE sch3.part1 PARTITION OF sch3.tmain FOR VALUES FROM (0) TO (5); +CREATE TABLE sch3.part2(id int) PARTITION BY RANGE(id); +CREATE FOREIGN TABLE sch3.part2_1 PARTITION OF sch3.part2 FOR VALUES FROM (5) TO (10) SERVER fdw_server; +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part2 FOR VALUES FROM (5) TO (10); + +-- Can't create publications with publish_via_partition_root = true, if table +-- has a partition that's a foreign table +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain WITH (publish_via_partition_root); +CREATE PUBLICATION pub1 FOR TABLES IN SCHEMA sch3 WITH (publish_via_partition_root); +CREATE PUBLICATION pub1 FOR ALL TABLES WITH (publish_via_partition_root); + +-- Test when a partitioned table with foreign table as a partition is attached +-- to partitioned table which is already published +ALTER TABLE sch3.tmain DETACH PARTITION sch3.part2; +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain WITH (publish_via_partition_root); +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part2 FOR VALUES FROM (5) TO (10); + +-- Can't create a foreign table that is partition of table published with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch3.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; + +-- Can't attach a foreign table as partition to table published with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_2(id int) SERVER fdw_server; +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); + +CREATE SCHEMA sch4; +CREATE TABLE sch4.tmain(id int) PARTITION BY RANGE(id); + +-- publication created with FOR TABLES IN SCHEMA +DROP PUBLICATION pub1; +CREATE PUBLICATION pub1 FOR TABLES IN SCHEMA sch4 WITH (publish_via_partition_root); + +-- Can't create a foreign table that is partition of table published with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch4.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; + +-- Can't attach a foreign table as partition to table published with +-- publish_via_partition_root = true +ALTER TABLE sch4.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); +DROP PUBLICATION pub1; + +-- Can't create publication with publish_via_partition_root = true on +-- partitioned table(which is not root) with a partition that's a foreign table +-- on other schema +CREATE SCHEMA sch5; +CREATE TABLE sch5.part1 PARTITION OF sch4.tmain FOR VALUES FROM (0) TO (10) PARTITION BY RANGE(id); +CREATE FOREIGN TABLE sch4.part1_1 PARTITION OF sch5.part1 FOR VALUES FROM (0) TO (5) SERVER fdw_server; +CREATE PUBLICATION pub1 FOR TABLES IN SCHEMA sch5 WITH (publish_via_partition_root); + +-- Test with publish_via_partition_root = false +-- Partition that are foreign tables are skipped by default +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain; +CREATE PUBLICATION pub2 FOR TABLES IN SCHEMA sch3; +CREATE PUBLICATION pub3 FOR ALL TABLES; + +-- Create a partition that's a foreign table of published table with +-- publish_via_partition_root = false +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch3.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; + +-- Attach partition that's a foreign table to published table +-- publish_via_partition_root = false +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); + +-- Check the published tables +SELECT pubname, tablename FROM pg_publication_tables WHERE schemaname in ('sch3', 'sch4') ORDER BY pubname, tablename; + +-- Can't alter publish_via_partition_root to true, if publication already have +-- a partition that's a foreign table +ALTER PUBLICATION pub1 SET (publish_via_partition_root); + +DROP PUBLICATION pub1; +DROP PUBLICATION pub2; +DROP PUBLICATION pub3; +DROP SCHEMA sch3 CASCADE; +DROP SCHEMA sch4 CASCADE; +DROP SCHEMA sch5 CASCADE; +DROP SERVER fdw_server; +DROP FOREIGN DATA WRAPPER test_fdw; + RESET SESSION AUTHORIZATION; DROP ROLE regress_publication_user, regress_publication_user2; DROP ROLE regress_publication_user_dummy; -- 2.34.1 ^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: Restrict publishing of partitioned table with a foreign table as partition 2025-04-28 12:48 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> @ 2025-04-28 14:27 ` Álvaro Herrera <[email protected]> 2025-05-09 14:55 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 0 siblings, 1 reply; 19+ messages in thread From: Álvaro Herrera @ 2025-04-28 14:27 UTC (permalink / raw) To: Shlok Kyal <[email protected]>; +Cc: vignesh C <[email protected]>; Sergey Tatarintsev <[email protected]>; [email protected] On 2025-Apr-28, Shlok Kyal wrote: > 2. > + * We also take a ShareLock on pg_partitioned_table to restrict addition > + * of new partitioned table which may contain a foreign partition while > + * publication is being created. XXX this is quite weird actually. > > This change was added to resolve the concurrency issue shared by > Vignesh in [4]. I tried with different locks and found that lock with > severity >= ShareLock was needed to avoid the concurrency issue. > Initially I added ShareLock to pg_class, but to reduce the scope I > added it to pg_partitioned_table instead. I cannot think of an > alternate approach. Do you have any suggestions for this? > [4]: https://www.postgresql.org/message-id/CALDaNm2%2BeL22Sbvj74uS37xvt%3DhaQWcOwP15QnDuVeYsjHiffw%40mail... I think this is the sticky point in this patch. I think you need a clearer explanation (in code comments) of why you need this lock, and whether a weaker lock would be enough in some cases (see below); also I suspect that these locking considerations are going to be important for users so they're going to need to be documented in the SGML docs. What operations are blocked when you hold this lock? Is replication going to block altogether until the transaction that runs publication_check_foreign_parts() commits/aborts? This is important because it might mean that that users need to keep such transactions short. If your publication is FOR TABLES IN SCHEMA, can you do with blocking creation of partitions only in that schema, or only partitions of partitioned tables in that schema? Another point that just occurred to me is that pg_upgrade --check may need to alert users that if they have an incompatible setup in 18 or earlier, then an upgrade to 19 does not work until they have fixed the publications, detached the partitions, or some other remediation has been applied. -- Álvaro Herrera Breisgau, Deutschland — https://www.EnterpriseDB.com/ "Linux transformó mi computadora, de una `máquina para hacer cosas', en un aparato realmente entretenido, sobre el cual cada día aprendo algo nuevo" (Jaime Salinas) ^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: Restrict publishing of partitioned table with a foreign table as partition 2025-04-28 12:48 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-04-28 14:27 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> @ 2025-05-09 14:55 ` Shlok Kyal <[email protected]> 2025-05-10 17:59 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> 0 siblings, 1 reply; 19+ messages in thread From: Shlok Kyal @ 2025-05-09 14:55 UTC (permalink / raw) To: Álvaro Herrera <[email protected]>; +Cc: vignesh C <[email protected]>; Sergey Tatarintsev <[email protected]>; [email protected] On Mon, 28 Apr 2025 at 19:57, Álvaro Herrera <[email protected]> wrote: > > On 2025-Apr-28, Shlok Kyal wrote: > > > 2. > > + * We also take a ShareLock on pg_partitioned_table to restrict addition > > + * of new partitioned table which may contain a foreign partition while > > + * publication is being created. XXX this is quite weird actually. > > > > This change was added to resolve the concurrency issue shared by > > Vignesh in [4]. I tried with different locks and found that lock with > > severity >= ShareLock was needed to avoid the concurrency issue. > > Initially I added ShareLock to pg_class, but to reduce the scope I > > added it to pg_partitioned_table instead. I cannot think of an > > alternate approach. Do you have any suggestions for this? > > > [4]: https://www.postgresql.org/message-id/CALDaNm2%2BeL22Sbvj74uS37xvt%3DhaQWcOwP15QnDuVeYsjHiffw%40mail... > > I think this is the sticky point in this patch. I think you need a > clearer explanation (in code comments) of why you need this lock, and > whether a weaker lock would be enough in some cases (see below); also I > suspect that these locking considerations are going to be important for > users so they're going to need to be documented in the SGML docs. What > operations are blocked when you hold this lock? Is replication going to > block altogether until the transaction that runs > publication_check_foreign_parts() commits/aborts? This is important > because it might mean that that users need to keep such transactions > short. ShareLock don't have any impact on replication. For a partitioned table here is the observation when a ShareLock is taken on the pg_partitioned table in a concurrent session: 1. CREATE TABLE (for partitioned table only) : blocked until sharelock is released 2. INSERT : No impact 3 .UPDATE : No impact 4. TRUNCATE : No impact 5. DELETE : No impact 6. VACUUM : blocked until sharelock is released 7. DROP TABLE : blocked until sharelock is released But I found one issue with the patch: Suppose we have only a partitioned table t1 in a database. Now we create a publication for all tables and add a breakpoint just after the 'ShareLock'. Now I run DROP TABLE t1; in a concurrent session. Now continue the CREATE PUBLICATION command. Now we hit a deadlock. logs: 2025-05-09 13:35:57.199 IST [3567096] ERROR: deadlock detected 2025-05-09 13:35:57.199 IST [3567096] DETAIL: Process 3567096 waits for AccessShareLock on relation 16411 of database 5; blocked by process 3567751. Process 3567751 waits for RowExclusiveLock on relation 3350 of database 5; blocked by process 3567096. Process 3567096: CREATE PUBLICATION pub1 FOR ALL TABLES WITH (publish_via_partition_root); Process 3567751: drop table t1; This happens because the DROP TABLE command takes an AccessExclusiveLock on the table t1 and is waiting to take RowExclusiveLock on pg_partitioned_table. And CREATE PUBLICATION command has taken ShareLock on pg_partitioned_table and is waiting to take an AccessShareLock on table t1. One thing I thought about is if we can change the ordering of the locks while checking during creating publication, but it is not possible as we must take lock on pg_partitoned_table first to avoid creation of any partitioned table while we are fetching the table list. Do you have suggestion what should we do in above case? > If your publication is FOR TABLES IN SCHEMA, can you do with blocking > creation of partitions only in that schema, or only partitions of > partitioned tables in that schema? > We should not block only a particular schema because we can create a partition of a partitioned table in a different schema as well. > Another point that just occurred to me is that pg_upgrade --check may > need to alert users that if they have an incompatible setup in 18 or > earlier, then an upgrade to 19 does not work until they have fixed the > publications, detached the partitions, or some other remediation has > been applied. > I have added a check in pg_upgrade and attached the same here. Thanks and Regards, Shlok Kyal Attachments: [application/octet-stream] v14-0001-Restrict-publishing-of-partitioned-table-with-fo.patch (39.8K, ../../CANhcyEVQ866NA5NbTi-aQhPj2zN_vbNb_ZW=E_P2foZYrW9VbA@mail.gmail.com/2-v14-0001-Restrict-publishing-of-partitioned-table-with-fo.patch) download | inline diff: From 9d7831fc9f160d3a6042f9a78d0cedbd3fedd546 Mon Sep 17 00:00:00 2001 From: Shlok Kyal <[email protected]> Date: Fri, 28 Mar 2025 11:15:09 +0530 Subject: [PATCH v14] Restrict publishing of partitioned table with foreign table as its partition Logical replication of foreign table is not supported and we throw an error in this case. But when we create a publication on a partitioned table that has a foreign table as its partition, the initial sync of such table is successful and we should avoid such cases. Current Behaviour in HEAD, when publication is created: 1. with publish_via_partition_root = true The root table is published, and initial data from partitions that are foreign tables is replicated. 2. with publish_via_partition_root = false and FOR ALL TABLES All leaf tables except partitions that are foreign tables are published. 3. with publish_via_partition_root = false and FOR TABLE/ FOR TABLES IN SCHEMA All leaf tables are published, including initial data from partitions that are foreign tables. With this patch we have following behaviour: 1. with publish_via_partition_root = true We throw an error when we try to publish a partition that is a foreign table. Error is thrown when we try to create a publication on (or add to existing publication) a partitioned table that has foreign tables as its partitions, when try to create a partition that is a foreign table and when we try to attach foreign table (or a partitioned table with foreign tables as partitions) to existing published tables. 2. with publish_via_partition_root = false We skip publishing partitions that are foreign tables. This is done by avoid adding such partitions in pg_subscription_rel catalog table. We have introduced two functions 'RelationHasForeignPartition' and 'publication_check_foreign_parts'. In 'RelationHasForeignPartition' we go through the child nodes of a partition and check if it has a foreign table. While doing so, we take an AccessShareLock on each partition table to prevent concurrent creation of a foreign table as a partition. In 'publication_check_foreign_parts' if schema id is provided we check for each partitioned table in that schema if it has a foreign table as its partition, or if schema id is not provided we check for each partitioned table in the database if it has a partition that's a foreign table. While doing so we take a ShareLock on pg_partitioned_table so no partition table is created concurrently after this check. --- doc/src/sgml/logical-replication.sgml | 10 +- doc/src/sgml/ref/create_publication.sgml | 18 +- src/backend/catalog/pg_publication.c | 220 ++++++++++++++++++++-- src/backend/commands/publicationcmds.c | 47 +++++ src/backend/commands/tablecmds.c | 112 +++++++++++ src/backend/partitioning/partdesc.c | 32 ++++ src/bin/pg_upgrade/check.c | 89 +++++++++ src/include/catalog/pg_publication.h | 3 + src/include/partitioning/partdesc.h | 1 + src/test/regress/expected/publication.out | 99 ++++++++++ src/test/regress/sql/publication.sql | 90 +++++++++ 11 files changed, 699 insertions(+), 22 deletions(-) diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index f288c049a5c..57236a452e9 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -2154,10 +2154,18 @@ CONTEXT: processing remote data for replication origin "pg_16395" during "INSER <listitem> <para> - Replication is only supported by tables, including partitioned tables. + Replication is only supported for tables, including partitioned tables. Attempts to replicate other types of relations, such as views, materialized views, or foreign tables, will result in an error. </para> + <para> + Replication is not supported for foreign tables. When used as partitions + of partitioned tables, publishing of the partitioned table is only allowed + if the <literal>publish_via_partition_root</literal> is set to + <literal>false</literal>. In this mode, changes to a partition that is a + foreign table are ignored for the purposes of replication, and data + contained in them is not included during initial synchronization. + </para> </listitem> <listitem> diff --git a/doc/src/sgml/ref/create_publication.sgml b/doc/src/sgml/ref/create_publication.sgml index 802630f2df1..1b07d57844e 100644 --- a/doc/src/sgml/ref/create_publication.sgml +++ b/doc/src/sgml/ref/create_publication.sgml @@ -251,13 +251,23 @@ CREATE PUBLICATION <replaceable class="parameter">name</replaceable> </para> <para> - This parameter also affects how row filters and column lists are - chosen for partitions; see below for details. + If this parameter is enabled, <literal>TRUNCATE</literal> + operations performed directly on partitions are not replicated. </para> <para> - If this is enabled, <literal>TRUNCATE</literal> operations performed - directly on partitions are not replicated. + If this parameter is enabled, foreign tables and partitioned tables + containing partitions that are foreign tables may not be + added to the publication. Conversely, foreign tables may not be + attached to a partitioned table that is included in a publication + with this parameter enabled. Lastly, this parameter may not be + changed on publications that include partitioned tables with foreign + tables as partitions. + </para> + + <para> + This parameter also affects how row filters and column lists are + chosen for partitions; see below for details. </para> </listitem> </varlistentry> diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c index d6f94db5d99..e6e453b097c 100644 --- a/src/backend/catalog/pg_publication.c +++ b/src/backend/catalog/pg_publication.c @@ -26,12 +26,15 @@ #include "catalog/partition.h" #include "catalog/pg_inherits.h" #include "catalog/pg_namespace.h" +#include "catalog/pg_partitioned_table.h" #include "catalog/pg_publication.h" #include "catalog/pg_publication_namespace.h" #include "catalog/pg_publication_rel.h" #include "catalog/pg_type.h" #include "commands/publicationcmds.h" #include "funcapi.h" +#include "partitioning/partdesc.h" +#include "storage/lmgr.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/catcache.h" @@ -53,7 +56,7 @@ typedef struct * error if not. */ static void -check_publication_add_relation(Relation targetrel) +check_publication_add_relation(Publication *pub, Relation targetrel) { /* Must be a regular or partitioned table */ if (RelationGetForm(targetrel)->relkind != RELKIND_RELATION && @@ -64,6 +67,21 @@ check_publication_add_relation(Relation targetrel) RelationGetRelationName(targetrel)), errdetail_relkind_not_supported(RelationGetForm(targetrel)->relkind))); + /* + * publish_via_partition_root cannot be true if it is a partitioned table + * and has any partition that's a foreign table. See + * publication_check_foreign_parts for details. + */ + if (pub->pubviaroot && + targetrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE && + RelationHasForeignPartition(targetrel)) + ereport(ERROR, + errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot add partitioned table \"%s\" to publication \"%s\", which has \"%s\" set to \"%s\"", + RelationGetRelationName(targetrel), pub->name, + "publish_via_partition_root", "true"), + errdetail("The table contains a partition that's a foreign table.")); + /* Can't be system table */ if (IsCatalogRelation(targetrel)) ereport(ERROR, @@ -111,6 +129,171 @@ check_publication_add_schema(Oid schemaid) errdetail("Temporary schemas cannot be replicated."))); } +/* + * Returns true if the ancestor is in the list of relations + * Otherwise, returns false. + */ +static bool +is_ancestor_member_relids(Oid ancestor, List *relids) +{ + foreach_oid(relid, relids) + { + if (relid == ancestor) + return true; + } + + return false; +} + +/* + * Filter out the partitions whose parent tables are also present in the list + * of relations. + */ +static void +filter_partition_rels(List *relids) +{ + foreach_oid(relid, relids) + { + bool skip = false; + List *ancestors = NIL; + + if (get_rel_relispartition(relid)) + ancestors = get_partition_ancestors(relid); + + foreach_oid(ancestor, ancestors) + { + if (is_ancestor_member_relids(ancestor, relids)) + { + skip = true; + break; + } + } + + if (skip) + relids = list_delete_oid(relids, relid); + } +} + +/* + * publication_check_foreign_parts + * Helper function to ensure we don't indirectly publish foreign tables + * + * DML data changes are not published for data in foreign tables, + * and yet the tablesync worker is not smart enough to omit data from + * foreign tables when they are partitions of partitioned tables. To + * avoid the inconsistencies that would result, we disallow foreign + * tables from being published generally. However, it's possible for + * partitioned tables to have foreign tables as partitions, and we would + * like to allow publishing those partitioned tables so that the other + * partitions are replicated. + * + * This function is in charge of detecting if a partitioned table has a + * foreign table as a partition -- either in the whole database (useful + * for FOR ALL TABLES publications) or in a particular schema (useful + * for FOR TABLES IN SCHEMA publications). This function must be called + * only for publications with publish_via_partition_root=true. + * + * When publish_via_partition_root is false, each partition published for + * replication is listed individually in pg_subscription_rel, and we + * don't add partitions that are foreign tables, so this check is not + * needed. + * + * If a valid schemaid is provided, check if that schema has any + * partitioned table with a foreign table as partition. + * + * If no valid schemaid is provided, check all partitioned tables. + * + * We take a lock on partition tables so no new foreign table are added + * concurrently as a partition. + * + * We also take a ShareLock on pg_partitioned_table to restrict addition + * of new partitioned table which may include a foreign table as partition + * while publication is being created. XXX this is quite weird actually. + */ +void +publication_check_foreign_parts(Oid schemaid, char *pubname) +{ + Relation classRel; + ScanKeyData key[3]; + int keycount = 0; + TableScanDesc scan; + HeapTuple tuple; + List *relids = NIL; + + /* + * Take lock on pg_partitioned_rel. This prevents new publications from + * being created. + */ + LockRelationOid(PartitionedRelationId, ShareLock); + + classRel = table_open(RelationRelationId, AccessShareLock); + + /* Get the root nodes of partitioned table */ + ScanKeyInit(&key[keycount++], + Anum_pg_class_relkind, + BTEqualStrategyNumber, F_CHAREQ, + CharGetDatum(RELKIND_PARTITIONED_TABLE)); + + /* If schema id is provided check partitioned table in that schema */ + if (OidIsValid(schemaid)) + ScanKeyInit(&key[keycount++], + Anum_pg_class_relnamespace, + BTEqualStrategyNumber, F_OIDEQ, + schemaid); + + /* + * If schema id is not provided, take relations for which relispartition + * is false. This will give only the root partitioned tables. We donot + * include this for the case when schema id is specified because there can + * be cases when root partitioned tables are not part of schema and one of + * the child partition can still have a foreign table as its partition. + */ + else + ScanKeyInit(&key[keycount++], + Anum_pg_class_relispartition, + BTEqualStrategyNumber, F_BOOLEQ, + BoolGetDatum(false)); + + scan = table_beginscan_catalog(classRel, keycount, key); + while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) + { + Form_pg_class relForm = (Form_pg_class) GETSTRUCT(tuple); + + relids = lappend_oid(relids, relForm->oid); + } + + /* + * If schema id is provided filter partitions list to have only topmost + * partitioned tables in that schema, to avoid repeated check. + */ + if (OidIsValid(schemaid)) + filter_partition_rels(relids); + + foreach_oid(relid, relids) + { + Relation pubrel = table_open(relid, AccessShareLock); + + if (pubrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE && + RelationHasForeignPartition(pubrel)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot set parameter \"%s\" to \"%s\" for publication \"%s\"", + "publish_via_partition_root", "true", pubname), + errtable(pubrel), + errdetail("Published partitioned table \"%s\" contains a partition that is a foreign table.", + get_rel_name(relid)))); + + /* + * Keep lock till end of transaction: must prevent this table from + * being attached a foreign table until we're done. + */ + table_close(pubrel, NoLock); + } + + table_endscan(scan); + table_close(classRel, AccessShareLock); +} + /* * Returns if relation represented by oid and Form_pg_class entry * is publishable. @@ -304,7 +487,7 @@ check_and_fetch_column_list(Publication *pub, Oid relid, MemoryContext mcxt, /* * Gets the relations based on the publication partition option for a specified - * relation. + * relation. Foreign tables are not included. */ List * GetPubPartitionOptionRelations(List *result, PublicationPartOpt pub_partopt, @@ -313,25 +496,21 @@ GetPubPartitionOptionRelations(List *result, PublicationPartOpt pub_partopt, if (get_rel_relkind(relid) == RELKIND_PARTITIONED_TABLE && pub_partopt != PUBLICATION_PART_ROOT) { - List *all_parts = find_all_inheritors(relid, NoLock, - NULL); + List *all_parts = find_all_inheritors(relid, NoLock, NULL); - if (pub_partopt == PUBLICATION_PART_ALL) - result = list_concat(result, all_parts); - else if (pub_partopt == PUBLICATION_PART_LEAF) + foreach_oid(partOid, all_parts) { - ListCell *lc; + char relkind = get_rel_relkind(partOid); - foreach(lc, all_parts) - { - Oid partOid = lfirst_oid(lc); + if (relkind == RELKIND_FOREIGN_TABLE) + continue; - if (get_rel_relkind(partOid) != RELKIND_PARTITIONED_TABLE) - result = lappend_oid(result, partOid); - } + if (pub_partopt == PUBLICATION_PART_LEAF && + relkind == RELKIND_PARTITIONED_TABLE) + continue; + + result = lappend_oid(result, partOid); } - else - Assert(false); } else result = lappend_oid(result, relid); @@ -463,7 +642,7 @@ publication_add_relation(Oid pubid, PublicationRelInfo *pri, RelationGetRelationName(targetrel), pub->name))); } - check_publication_add_relation(targetrel); + check_publication_add_relation(pub, targetrel); /* Validate and translate column names into a Bitmapset of attnums. */ attnums = pub_collist_validate(pri->relation, pri->columns); @@ -703,6 +882,13 @@ publication_add_schema(Oid pubid, Oid schemaid, bool if_not_exists) check_publication_add_schema(schemaid); + /* + * If publish_via_partition_root is true, check if schema has any foreign + * partition + */ + if (pub->pubviaroot) + publication_check_foreign_parts(schemaid, pub->name); + /* Form a tuple */ memset(values, 0, sizeof(values)); memset(nulls, false, sizeof(nulls)); diff --git a/src/backend/commands/publicationcmds.c b/src/backend/commands/publicationcmds.c index 0b23d94c38e..a2411fbebda 100644 --- a/src/backend/commands/publicationcmds.c +++ b/src/backend/commands/publicationcmds.c @@ -35,6 +35,7 @@ #include "commands/publicationcmds.h" #include "miscadmin.h" #include "nodes/nodeFuncs.h" +#include "partitioning/partdesc.h" #include "parser/parse_clause.h" #include "parser/parse_collate.h" #include "parser/parse_relation.h" @@ -915,6 +916,12 @@ CreatePublication(ParseState *pstate, CreatePublicationStmt *stmt) /* Associate objects with the publication. */ if (stmt->for_all_tables) { + /* + * Check if any partitioned table has a foreign table as its partition + */ + if (publish_via_partition_root) + publication_check_foreign_parts(InvalidOid, stmt->pubname); + /* Invalidate relcache so that publication info is rebuilt. */ CacheInvalidateRelcacheAll(); } @@ -1080,6 +1087,46 @@ AlterPublicationOptions(ParseState *pstate, AlterPublicationStmt *stmt, } } + /* + * If publish_via_partition_root is set to true, check if the publication + * has any partition that's a foreign table. See + * publication_check_foreign_parts for details. + */ + if (publish_via_partition_root_given && publish_via_partition_root) + { + char *pubname = stmt->pubname; + List *schemaoids; + List *relids; + + if (pubform->puballtables) + publication_check_foreign_parts(InvalidOid, pubname); + + schemaoids = GetPublicationSchemas(pubform->oid); + foreach_oid(schemaoid, schemaoids) + publication_check_foreign_parts(schemaoid, pubname); + + relids = GetPublicationRelations(pubform->oid, PUBLICATION_PART_ROOT); + foreach_oid(relid, relids) + { + Relation pubrel; + + if (get_rel_relkind(relid) != RELKIND_PARTITIONED_TABLE) + continue; + + pubrel = table_open(relid, AccessShareLock); + + if (RelationHasForeignPartition(pubrel)) + ereport(ERROR, + errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot set parameter \"%s\" to \"%s\" for publication \"%s\"", + "publish_via_partition_root", "true", pubname), + errdetail("Published partitioned table \"%s\" contains a partition that is a foreign table.", + RelationGetRelationName(pubrel))); + + table_close(pubrel, NoLock); + } + } + /* Everything ok, form a new tuple. */ memset(values, 0, sizeof(values)); memset(nulls, false, sizeof(nulls)); diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 54ad38247aa..9e94a489cd6 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -1133,6 +1133,53 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, errmsg("\"%s\" is not partitioned", RelationGetRelationName(parent)))); + /* + * If we're creating a partition that's a foreign table, verify that + * the parent table is not in a publication with + * publish_via_partition_root enabled. For details, see + * publication_check_foreign_parts. + */ + if (rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE) + { + Oid schemaid; + List *puboids; + List *ancestors; + + /* Start with publications of all tables */ + puboids = GetAllTablesPublications(); + + /* capture all publications that include this relation directly */ + puboids = GetRelationPublications(parent->rd_id); + schemaid = RelationGetNamespace(parent); + puboids = list_concat(puboids, GetSchemaPublications(schemaid)); + + /* and do the same for its ancestors, if any */ + ancestors = get_partition_ancestors(parent->rd_id); + foreach_oid(ancestor, ancestors) + { + puboids = list_concat(puboids, GetRelationPublications(ancestor)); + schemaid = get_rel_namespace(ancestor); + puboids = list_concat(puboids, GetSchemaPublications(schemaid)); + } + + /* Check the publish_via_partition_root bit for each of those */ + list_sort(puboids, list_oid_cmp); + list_deduplicate_oid(puboids); + foreach_oid(puboid, puboids) + { + Publication *pub = GetPublication(puboid); + + if (pub->pubviaroot) + ereport(ERROR, + errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot create foreign table \"%s\" as a partition of \"%s\"", + RelationGetRelationName(rel), RelationGetRelationName(parent)), + errdetail("Partitioned table \"%s\" is published with option \"%s\" in publication \"%s\".", + RelationGetRelationName(parent), + "publish_via_partition_root", pub->name)); + } + } + /* * The partition constraint of the default partition depends on the * partition bounds of every other partition. It is possible that @@ -20291,6 +20338,71 @@ ATExecAttachPartition(List **wqueue, Relation rel, PartitionCmd *cmd, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot attach temporary relation of another session as partition"))); + /* + * If the relation to attach is a foreign table, or a partitioned table + * that contains a foreign table as partition, then verify that the parent + * table is not in a publication with publish_via_partition_root enabled. + * See publication_check_foreign_parts. + */ + if (attachrel->rd_rel->relkind == RELKIND_FOREIGN_TABLE || + (attachrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE && + RelationHasForeignPartition(attachrel))) + { + Oid schemaid; + List *puboids; + List *ancestors; + + /* Start with publications of all tables */ + puboids = GetAllTablesPublications(); + + /* capture all publications that include this relation directly */ + puboids = list_concat(puboids, GetRelationPublications(rel->rd_id)); + schemaid = RelationGetNamespace(rel); + puboids = list_concat(puboids, GetSchemaPublications(schemaid)); + + /* and do the same for its ancestors, if any */ + ancestors = get_partition_ancestors(rel->rd_id); + foreach_oid(ancestor, ancestors) + { + puboids = list_concat(puboids, GetRelationPublications(ancestor)); + schemaid = get_rel_namespace(ancestor); + puboids = list_concat(puboids, GetSchemaPublications(schemaid)); + } + + /* Now check the publish_via_partition_root bit for each of those */ + list_sort(puboids, list_oid_cmp); + list_deduplicate_oid(puboids); + foreach_oid(puboid, puboids) + { + Publication *pub; + + pub = GetPublication(puboid); + if (pub->pubviaroot) + { + if (attachrel->rd_rel->relkind == RELKIND_FOREIGN_TABLE) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot attach foreign table \"%s\" to partition table \"%s\"", + RelationGetRelationName(attachrel), + RelationGetRelationName(rel)), + errdetail("Partitioned table \"%s\" is published with option \"%s\" in publication \"%s\".", + RelationGetRelationName(rel), + "publish_via_partition_root", + pub->name))); + else + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot attach table \"%s\" with a partition that's a foreign table to partition table \"%s\"", + RelationGetRelationName(attachrel), + RelationGetRelationName(rel)), + errdetail("Partitioned table \"%s\" is published with option \"%s\" in publication \"%s\".", + RelationGetRelationName(rel), + "publish_via_partition_root", + pub->name))); + } + } + } + /* * Check if attachrel has any identity columns or any columns that aren't * in the parent. diff --git a/src/backend/partitioning/partdesc.c b/src/backend/partitioning/partdesc.c index 328b4d450e4..b53139bafdd 100644 --- a/src/backend/partitioning/partdesc.c +++ b/src/backend/partitioning/partdesc.c @@ -506,3 +506,35 @@ get_default_oid_from_partdesc(PartitionDesc partdesc) return InvalidOid; } + +/* + * Return true if the given partitioned table ultimately contains a + * partition that is a foreign table, false otherwise. + */ +bool +RelationHasForeignPartition(Relation rel) +{ + PartitionDesc pd = RelationGetPartitionDesc(rel, true); + + for (int i = 0; i < pd->nparts; i++) + { + if (pd->is_leaf[i]) + { + if (get_rel_relkind(pd->oids[i]) == RELKIND_FOREIGN_TABLE) + return true; + } + else + { + Relation part; + bool ret; + + part = table_open(pd->oids[i], AccessShareLock); + ret = RelationHasForeignPartition(part); + table_close(part, NoLock); + if (ret) + return true; + } + } + + return false; +} diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index 940fc77fc2e..2cf407a3169 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -31,6 +31,7 @@ static void check_new_cluster_logical_replication_slots(void); static void check_new_cluster_subscription_configuration(void); static void check_old_cluster_for_valid_slots(void); static void check_old_cluster_subscription_state(void); +static void check_for_valid_publication(ClusterInfo *cluster); /* * DataTypesUsageChecks - definitions of data type checks for the old cluster @@ -633,6 +634,8 @@ check_and_dump_old_cluster(void) check_old_cluster_subscription_state(); } + check_for_valid_publication(&old_cluster); + check_for_data_types_usage(&old_cluster); /* @@ -2272,3 +2275,89 @@ check_old_cluster_subscription_state(void) else check_ok(); } + +/* + * Callback function for processing results of query for + * check_for_valid_publication()'s UpgradeTask. If the query returned + * any rows (i.e., the check failed), write the details to the report file. + */ +static void +process_pub_check(DbInfo *dbinfo, PGresult *res, void *arg) +{ + UpgradeTaskReport *report = (UpgradeTaskReport *) arg; + int ntups = PQntuples(res); + int i_puboid = PQfnumber(res, "oid"); + int i_pubviaroot = PQfnumber(res, "pubviaroot"); + int i_puballtables = PQfnumber(res, "puballtables"); + int i_ftrelid = PQfnumber(res, "ftrelid"); + + if (ntups == 0) + return; + + if (report->file == NULL && + (report->file = fopen_priv(report->path, "w")) == NULL) + pg_fatal("could not open file \"%s\": %m", report->path); + + fprintf(report->file, "In database: %s\n", dbinfo->db_name); + + for (int rowno = 0; rowno < ntups; rowno++) + fprintf(report->file, "The publication with oid: %s, has foreign table with oid: %s as partition of a published table. " + "Publication is published with publish_via_partition_root = %s and it is published %s.\n", + PQgetvalue(res, rowno, i_puboid), + PQgetvalue(res, rowno, i_ftrelid), + strcmp(PQgetvalue(res, rowno, i_pubviaroot), "f") ? "true" : "false", + strcmp(PQgetvalue(res, rowno, i_puballtables), "f") ? "FOR ALL TABLES" : "FOR TABLE/FOR TABLES IN SCHEMA"); +} + +/* + * check_for_valid_publication + * + * Verify if the publication has any published foreign table which are + * partitions of a table. + * + * For PG 18 and below foreign tables can be indirectly published when it is + * partition of a partitioned table. Avoid upgrading the cluster if it has a + * publication with publish_via_partition_root = true and has foreign table + * which is partition of a published table. + */ +static void +check_for_valid_publication(ClusterInfo *cluster) +{ + UpgradeTaskReport report; + UpgradeTask *task; + const char *query; + + if (GET_MAJOR_VERSION(cluster->major_version) >= 1900) + return; + + prep_status("Checking for invalid publications"); + + report.file = NULL; + snprintf(report.path, sizeof(report.path), "%s/%s", + log_opts.basedir, + "pubs_invalid.txt"); + + query = "SELECT p.oid, p.pubviaroot, p.puballtables, ft.ftrelid " + "from pg_catalog.pg_publication p, LATERAL " + "pg_get_publication_tables(p.pubname) gpt, LATERAL " + "pg_partition_tree(gpt.relid) gt JOIN pg_catalog.pg_foreign_table ft ON " + "ft.ftrelid = gt.relid WHERE p.pubviaroot = true;"; + + task = upgrade_task_create(); + upgrade_task_add_step(task, query, + process_pub_check, + true, &report); + upgrade_task_run(task, cluster); + upgrade_task_free(task); + + if (report.file) + { + fclose(report.file); + pg_log(PG_REPORT, "fatal"); + pg_fatal("Your installation contains publications, which has foreign table which are partitions of published table.\n" + "A list of potentially-affected publications is in the file:\n" + " %s", report.path); + } + else + check_ok(); +} diff --git a/src/include/catalog/pg_publication.h b/src/include/catalog/pg_publication.h index 48c7d1a8615..71ad5a6f846 100644 --- a/src/include/catalog/pg_publication.h +++ b/src/include/catalog/pg_publication.h @@ -19,6 +19,7 @@ #include "catalog/genbki.h" #include "catalog/objectaddress.h" +#include "catalog/pg_class.h" #include "catalog/pg_publication_d.h" /* IWYU pragma: export */ /* ---------------- @@ -178,6 +179,8 @@ extern Oid GetTopMostAncestorInPublication(Oid puboid, List *ancestors, extern bool is_publishable_relation(Relation rel); extern bool is_schema_publication(Oid pubid); +extern void publication_check_foreign_parts(Oid schemaid, char *pubname); + extern bool check_and_fetch_column_list(Publication *pub, Oid relid, MemoryContext mcxt, Bitmapset **cols); extern ObjectAddress publication_add_relation(Oid pubid, PublicationRelInfo *pri, diff --git a/src/include/partitioning/partdesc.h b/src/include/partitioning/partdesc.h index 34533f7004c..5fbafdc06f9 100644 --- a/src/include/partitioning/partdesc.h +++ b/src/include/partitioning/partdesc.h @@ -71,5 +71,6 @@ extern PartitionDesc PartitionDirectoryLookup(PartitionDirectory, Relation); extern void DestroyPartitionDirectory(PartitionDirectory pdir); extern Oid get_default_oid_from_partdesc(PartitionDesc partdesc); +extern bool RelationHasForeignPartition(Relation rel); #endif /* PARTDESC_H */ diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out index 4de96c04f9d..f57addab23d 100644 --- a/src/test/regress/expected/publication.out +++ b/src/test/regress/expected/publication.out @@ -1924,6 +1924,105 @@ DROP PUBLICATION pub1; DROP PUBLICATION pub2; DROP TABLE gencols; RESET client_min_messages; +-- ====================================================== +-- Test when foreign table is a partition of a partitioned table on which +-- publication is created +SET client_min_messages = 'ERROR'; +CREATE FOREIGN DATA WRAPPER test_fdw; +CREATE SERVER fdw_server FOREIGN DATA WRAPPER test_fdw; +CREATE SCHEMA sch3; +CREATE TABLE sch3.tmain(id int) PARTITION BY RANGE(id); +CREATE TABLE sch3.part1 PARTITION OF sch3.tmain FOR VALUES FROM (0) TO (5); +CREATE TABLE sch3.part2(id int) PARTITION BY RANGE(id); +CREATE FOREIGN TABLE sch3.part2_1 PARTITION OF sch3.part2 FOR VALUES FROM (5) TO (10) SERVER fdw_server; +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part2 FOR VALUES FROM (5) TO (10); +-- Can't create publications with publish_via_partition_root = true, if table +-- has a partition that's a foreign table +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain WITH (publish_via_partition_root); +ERROR: cannot add partitioned table "tmain" to publication "pub1", which has "publish_via_partition_root" set to "true" +DETAIL: The table contains a partition that's a foreign table. +CREATE PUBLICATION pub1 FOR TABLES IN SCHEMA sch3 WITH (publish_via_partition_root); +ERROR: cannot set parameter "publish_via_partition_root" to "true" for publication "pub1" +DETAIL: Published partitioned table "tmain" contains a partition that is a foreign table. +CREATE PUBLICATION pub1 FOR ALL TABLES WITH (publish_via_partition_root); +ERROR: cannot set parameter "publish_via_partition_root" to "true" for publication "pub1" +DETAIL: Published partitioned table "tmain" contains a partition that is a foreign table. +-- Test when a partitioned table with foreign table as a partition is attached +-- to partitioned table which is already published +ALTER TABLE sch3.tmain DETACH PARTITION sch3.part2; +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain WITH (publish_via_partition_root); +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part2 FOR VALUES FROM (5) TO (10); +ERROR: cannot attach table "part2" with a partition that's a foreign table to partition table "tmain" +DETAIL: Partitioned table "tmain" is published with option "publish_via_partition_root" in publication "pub1". +-- Can't create a foreign table that is partition of table published with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch3.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; +ERROR: cannot create foreign table "part3_1" as a partition of "tmain" +DETAIL: Partitioned table "tmain" is published with option "publish_via_partition_root" in publication "pub1". +-- Can't attach a foreign table as partition to table published with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_2(id int) SERVER fdw_server; +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); +ERROR: cannot attach foreign table "part3_2" to partition table "tmain" +DETAIL: Partitioned table "tmain" is published with option "publish_via_partition_root" in publication "pub1". +CREATE SCHEMA sch4; +CREATE TABLE sch4.tmain(id int) PARTITION BY RANGE(id); +-- publication created with FOR TABLES IN SCHEMA +DROP PUBLICATION pub1; +CREATE PUBLICATION pub1 FOR TABLES IN SCHEMA sch4 WITH (publish_via_partition_root); +-- Can't create a foreign table that is partition of table published with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch4.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; +ERROR: cannot create foreign table "part3_1" as a partition of "tmain" +DETAIL: Partitioned table "tmain" is published with option "publish_via_partition_root" in publication "pub1". +-- Can't attach a foreign table as partition to table published with +-- publish_via_partition_root = true +ALTER TABLE sch4.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); +ERROR: cannot attach foreign table "part3_2" to partition table "tmain" +DETAIL: Partitioned table "tmain" is published with option "publish_via_partition_root" in publication "pub1". +DROP PUBLICATION pub1; +-- Can't create publication with publish_via_partition_root = true on +-- partitioned table(which is not root) with a partition that's a foreign table +-- on other schema +CREATE SCHEMA sch5; +CREATE TABLE sch5.part1 PARTITION OF sch4.tmain FOR VALUES FROM (0) TO (10) PARTITION BY RANGE(id); +CREATE FOREIGN TABLE sch4.part1_1 PARTITION OF sch5.part1 FOR VALUES FROM (0) TO (5) SERVER fdw_server; +CREATE PUBLICATION pub1 FOR TABLES IN SCHEMA sch5 WITH (publish_via_partition_root); +ERROR: cannot set parameter "publish_via_partition_root" to "true" for publication "pub1" +DETAIL: Published partitioned table "part1" contains a partition that is a foreign table. +-- Test with publish_via_partition_root = false +-- Partition that are foreign tables are skipped by default +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain; +CREATE PUBLICATION pub2 FOR TABLES IN SCHEMA sch3; +CREATE PUBLICATION pub3 FOR ALL TABLES; +-- Create a partition that's a foreign table of published table with +-- publish_via_partition_root = false +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch3.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; +-- Attach partition that's a foreign table to published table +-- publish_via_partition_root = false +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); +-- Check the published tables +SELECT pubname, tablename FROM pg_publication_tables WHERE schemaname in ('sch3', 'sch4') ORDER BY pubname, tablename; + pubname | tablename +---------+----------- + pub1 | part1 + pub2 | part1 + pub3 | part1 +(3 rows) + +-- Can't alter publish_via_partition_root to true, if publication already have +-- a partition that's a foreign table +ALTER PUBLICATION pub1 SET (publish_via_partition_root); +ERROR: cannot set parameter "publish_via_partition_root" to "true" for publication "pub1" +DETAIL: Published partitioned table "tmain" contains a partition that is a foreign table. +DROP PUBLICATION pub1; +DROP PUBLICATION pub2; +DROP PUBLICATION pub3; +DROP SCHEMA sch3 CASCADE; +DROP SCHEMA sch4 CASCADE; +DROP SCHEMA sch5 CASCADE; +DROP SERVER fdw_server; +DROP FOREIGN DATA WRAPPER test_fdw; RESET SESSION AUTHORIZATION; DROP ROLE regress_publication_user, regress_publication_user2; DROP ROLE regress_publication_user_dummy; diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql index 68001de4000..55c0dff3601 100644 --- a/src/test/regress/sql/publication.sql +++ b/src/test/regress/sql/publication.sql @@ -1223,6 +1223,96 @@ DROP PUBLICATION pub2; DROP TABLE gencols; RESET client_min_messages; +-- ====================================================== + +-- Test when foreign table is a partition of a partitioned table on which +-- publication is created +SET client_min_messages = 'ERROR'; +CREATE FOREIGN DATA WRAPPER test_fdw; +CREATE SERVER fdw_server FOREIGN DATA WRAPPER test_fdw; + +CREATE SCHEMA sch3; +CREATE TABLE sch3.tmain(id int) PARTITION BY RANGE(id); +CREATE TABLE sch3.part1 PARTITION OF sch3.tmain FOR VALUES FROM (0) TO (5); +CREATE TABLE sch3.part2(id int) PARTITION BY RANGE(id); +CREATE FOREIGN TABLE sch3.part2_1 PARTITION OF sch3.part2 FOR VALUES FROM (5) TO (10) SERVER fdw_server; +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part2 FOR VALUES FROM (5) TO (10); + +-- Can't create publications with publish_via_partition_root = true, if table +-- has a partition that's a foreign table +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain WITH (publish_via_partition_root); +CREATE PUBLICATION pub1 FOR TABLES IN SCHEMA sch3 WITH (publish_via_partition_root); +CREATE PUBLICATION pub1 FOR ALL TABLES WITH (publish_via_partition_root); + +-- Test when a partitioned table with foreign table as a partition is attached +-- to partitioned table which is already published +ALTER TABLE sch3.tmain DETACH PARTITION sch3.part2; +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain WITH (publish_via_partition_root); +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part2 FOR VALUES FROM (5) TO (10); + +-- Can't create a foreign table that is partition of table published with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch3.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; + +-- Can't attach a foreign table as partition to table published with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_2(id int) SERVER fdw_server; +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); + +CREATE SCHEMA sch4; +CREATE TABLE sch4.tmain(id int) PARTITION BY RANGE(id); + +-- publication created with FOR TABLES IN SCHEMA +DROP PUBLICATION pub1; +CREATE PUBLICATION pub1 FOR TABLES IN SCHEMA sch4 WITH (publish_via_partition_root); + +-- Can't create a foreign table that is partition of table published with +-- publish_via_partition_root = true +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch4.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; + +-- Can't attach a foreign table as partition to table published with +-- publish_via_partition_root = true +ALTER TABLE sch4.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); +DROP PUBLICATION pub1; + +-- Can't create publication with publish_via_partition_root = true on +-- partitioned table(which is not root) with a partition that's a foreign table +-- on other schema +CREATE SCHEMA sch5; +CREATE TABLE sch5.part1 PARTITION OF sch4.tmain FOR VALUES FROM (0) TO (10) PARTITION BY RANGE(id); +CREATE FOREIGN TABLE sch4.part1_1 PARTITION OF sch5.part1 FOR VALUES FROM (0) TO (5) SERVER fdw_server; +CREATE PUBLICATION pub1 FOR TABLES IN SCHEMA sch5 WITH (publish_via_partition_root); + +-- Test with publish_via_partition_root = false +-- Partition that are foreign tables are skipped by default +CREATE PUBLICATION pub1 FOR TABLE sch3.tmain; +CREATE PUBLICATION pub2 FOR TABLES IN SCHEMA sch3; +CREATE PUBLICATION pub3 FOR ALL TABLES; + +-- Create a partition that's a foreign table of published table with +-- publish_via_partition_root = false +CREATE FOREIGN TABLE sch3.part3_1 PARTITION OF sch3.tmain FOR VALUES FROM (10) TO (15) SERVER fdw_server; + +-- Attach partition that's a foreign table to published table +-- publish_via_partition_root = false +ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part3_2 FOR VALUES FROM (15) TO (20); + +-- Check the published tables +SELECT pubname, tablename FROM pg_publication_tables WHERE schemaname in ('sch3', 'sch4') ORDER BY pubname, tablename; + +-- Can't alter publish_via_partition_root to true, if publication already have +-- a partition that's a foreign table +ALTER PUBLICATION pub1 SET (publish_via_partition_root); + +DROP PUBLICATION pub1; +DROP PUBLICATION pub2; +DROP PUBLICATION pub3; +DROP SCHEMA sch3 CASCADE; +DROP SCHEMA sch4 CASCADE; +DROP SCHEMA sch5 CASCADE; +DROP SERVER fdw_server; +DROP FOREIGN DATA WRAPPER test_fdw; + RESET SESSION AUTHORIZATION; DROP ROLE regress_publication_user, regress_publication_user2; DROP ROLE regress_publication_user_dummy; -- 2.34.1 ^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: Restrict publishing of partitioned table with a foreign table as partition 2025-04-28 12:48 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-04-28 14:27 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> 2025-05-09 14:55 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> @ 2025-05-10 17:59 ` Álvaro Herrera <[email protected]> 2025-05-15 12:48 ` Re: Restrict publishing of partitioned table with a foreign table as partition Amit Kapila <[email protected]> 0 siblings, 1 reply; 19+ messages in thread From: Álvaro Herrera @ 2025-05-10 17:59 UTC (permalink / raw) To: Shlok Kyal <[email protected]>; +Cc: vignesh C <[email protected]>; Sergey Tatarintsev <[email protected]>; [email protected] Hello, I think reimplementing list_member_oid() under a different name (is_ancestor_member_relids) is pointless and should not be done. It also appears to me that we haven't nailed the error messages just yet. I tried to fix it upthread, but didn't really get it correct. For instance, consider this case taken from the new regression tests: /* test setup */ CREATE SCHEMA sch3; CREATE TABLE sch3.tmain(id int) PARTITION BY RANGE(id); CREATE TABLE sch3.part1 PARTITION OF sch3.tmain FOR VALUES FROM (0) TO (5); CREATE TABLE sch3.part2(id int) PARTITION BY RANGE(id); CREATE FOREIGN TABLE sch3.part2_1 PARTITION OF sch3.part2 FOR VALUES FROM (5) TO (10) SERVER fdw_server; ALTER TABLE sch3.tmain ATTACH PARTITION sch3.part2 FOR VALUES FROM (5) TO (10); The issue occurs in the next command: CREATE PUBLICATION pub1 FOR TABLE sch3.tmain WITH (publish_via_partition_root); ERROR: cannot add partitioned table "tmain" to publication "pub1", which has "publish_via_partition_root" set to "true" DETAIL: The table contains a partition that's a foreign table. The user is trying to create a publication, so an error saying that it's not possible to add a partition is a bit off-point. (If you squint hard enough it sort of makes sense, but it seems inconsistent enough.) We could replace the word "add" with the word "include" there and this particular problem would be fixed, perhaps. However, the next one is worse: CREATE PUBLICATION pub1 FOR ALL TABLES WITH (publish_via_partition_root); ERROR: cannot set parameter "publish_via_partition_root" to "true" for publication "pub1" DETAIL: Published partitioned table "tmain" contains a partition that is a foreign table. Here we again try to create a publication, but we get an error that we cannot set the parameter, which is even more detached from reality. Maybe what we need is to add another parameter to that function to indicate which operation is being executed (create publication, add schema, change the parameter), which is used to indicate the error message to throw. With that thought in mind, I think it would make things simpler for us to remove the duplicate copies of the same error messages by adding yet another function, after publication_check_foreign_parts(), which receives the operation being executed and whatever additional parameter it needs, and throws the error. If publication_check_foreign_parts() also receives the operation being executed, it can simply pass it through to that new function when an error needs to be thrown; other places that hardcode error situations would pass a constant for the operation. But the non-idiomatic locking of pg_partitioned_table appears to continue to be the pain point of this patch. My impression is that using a lock is the wrong approach to solve the concurrency problem. Maybe we can use a ConditionVariable instead somehow. (The real trick here is to figure out exactly _how_ to use the CV, I mean what exactly is the condition that the CV sleeps on?) -- Álvaro Herrera 48°01'N 7°57'E — https://www.EnterpriseDB.com/ "Industry suffers from the managerial dogma that for the sake of stability and continuity, the company should be independent of the competence of individual employees." (E. Dijkstra) ^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: Restrict publishing of partitioned table with a foreign table as partition 2025-04-28 12:48 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-04-28 14:27 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> 2025-05-09 14:55 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-05-10 17:59 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> @ 2025-05-15 12:48 ` Amit Kapila <[email protected]> 2025-05-19 16:32 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 0 siblings, 1 reply; 19+ messages in thread From: Amit Kapila @ 2025-05-15 12:48 UTC (permalink / raw) To: Álvaro Herrera <[email protected]>; +Cc: Shlok Kyal <[email protected]>; vignesh C <[email protected]>; Sergey Tatarintsev <[email protected]>; [email protected] On Sun, May 11, 2025 at 6:53 AM Álvaro Herrera <[email protected]> wrote: > > But the non-idiomatic locking of pg_partitioned_table appears to > continue to be the pain point of this patch. My impression is that > using a lock is the wrong approach to solve the concurrency problem. > Maybe we can use a ConditionVariable instead somehow. (The real trick > here is to figure out exactly _how_ to use the CV, I mean what exactly > is the condition that the CV sleeps on?) > Can we fix this problem by having a check at the time of CREATESUBSCRIPTION such that, if copy-data is true, then we ensure that the specified publishers don't have a foreign table? We have a somewhat similar check for publications in the function check_publications_origin(), though for a different purpose. Along with it, we can still have a check during foreign table creation/attach that it doesn't become part of some publication, as the patch may have it now. -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: Restrict publishing of partitioned table with a foreign table as partition 2025-04-28 12:48 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-04-28 14:27 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> 2025-05-09 14:55 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-05-10 17:59 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> 2025-05-15 12:48 ` Re: Restrict publishing of partitioned table with a foreign table as partition Amit Kapila <[email protected]> @ 2025-05-19 16:32 ` Shlok Kyal <[email protected]> 2025-06-04 10:41 ` Re: Restrict publishing of partitioned table with a foreign table as partition Ajin Cherian <[email protected]> 0 siblings, 1 reply; 19+ messages in thread From: Shlok Kyal @ 2025-05-19 16:32 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: Álvaro Herrera <[email protected]>; vignesh C <[email protected]>; Sergey Tatarintsev <[email protected]>; [email protected] On Thu, 15 May 2025 at 18:19, Amit Kapila <[email protected]> wrote: > > On Sun, May 11, 2025 at 6:53 AM Álvaro Herrera <[email protected]> wrote: > > > > But the non-idiomatic locking of pg_partitioned_table appears to > > continue to be the pain point of this patch. My impression is that > > using a lock is the wrong approach to solve the concurrency problem. > > Maybe we can use a ConditionVariable instead somehow. (The real trick > > here is to figure out exactly _how_ to use the CV, I mean what exactly > > is the condition that the CV sleeps on?) > > > > Can we fix this problem by having a check at the time of > CREATESUBSCRIPTION such that, if copy-data is true, then we ensure > that the specified publishers don't have a foreign table? We have a > somewhat similar check for publications in the function > check_publications_origin(), though for a different purpose. Along > with it, we can still have a check during foreign table > creation/attach that it doesn't become part of some publication, as > the patch may have it now. > This approach seems better to me. I have created a patch with the above approach. Thanks and Regards, Shlok Kyal Attachments: [application/octet-stream] v15-0001-Restrict-publishing-of-partitioned-table-with-fo.patch (29.1K, ../../CANhcyEVUf5PEmahQvcc8GgCzxaAnJ8kTrir0Am_7O4SkZnpkQA@mail.gmail.com/2-v15-0001-Restrict-publishing-of-partitioned-table-with-fo.patch) download | inline diff: From f13809f9a46d18eec02407cf6ea4a104627e9a9c Mon Sep 17 00:00:00 2001 From: Shlok Kyal <[email protected]> Date: Mon, 19 May 2025 19:47:04 +0530 Subject: [PATCH v15] Restrict publishing of partitioned table with foreign table as its partition Logical replication of foreign table is not supported and we throw an error in this case. But when we create a publication on a partitioned table that has a foreign table as its partition, the initial sync of such table is successful and we should avoid such cases. Current Behaviour in HEAD, when publication is created: 1. with publish_via_partition_root = true The root table is published, and initial data from partitions that are foreign tables is replicated. 2. with publish_via_partition_root = false and FOR ALL TABLES All leaf tables except partitions that are foreign tables are published. 3. with publish_via_partition_root = false and FOR TABLE/ FOR TABLES IN SCHEMA All leaf tables are published, including initial data from partitions that are foreign tables. With this patch we have following behaviour: 1. with publish_via_partition_root = true An error is thrown when we try to create subscription (with copy_data = true) on publication with partitioned table with foreign partition. When copy_data = false, we donot need this check as initial sync is not performed. Also we restrict creating/attaching a foreign table as partition of a partitioned table which is part of any publication. 2. with publish_via_partition_root = false We skip publishing partitions that are foreign tables. This is done by avoid adding such partitions in pg_subscription_rel catalog table. We have introduced two functions 'RelationHasForeignPartition' and 'check_publications_foreign_parts'. In 'RelationHasForeignPartition' we go through the child nodes of a partition and check if it has a foreign table. In 'check_publications_foreign_parts' we check if a subscription with copy_data = true is subscribing to a publication which has partitioned table with foreign table as its partition. --- doc/src/sgml/logical-replication.sgml | 10 +- doc/src/sgml/ref/create_subscription.sgml | 7 ++ src/backend/catalog/pg_publication.c | 26 ++-- src/backend/commands/subscriptioncmds.c | 101 ++++++++++++++++ src/backend/commands/tablecmds.c | 110 +++++++++++++++++ src/backend/partitioning/partdesc.c | 32 +++++ src/bin/pg_upgrade/check.c | 89 ++++++++++++++ src/include/partitioning/partdesc.h | 1 + src/test/subscription/Makefile | 1 + src/test/subscription/t/013_partition.pl | 139 ++++++++++++++++++++++ 10 files changed, 500 insertions(+), 16 deletions(-) diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index f288c049a5c..57236a452e9 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -2154,10 +2154,18 @@ CONTEXT: processing remote data for replication origin "pg_16395" during "INSER <listitem> <para> - Replication is only supported by tables, including partitioned tables. + Replication is only supported for tables, including partitioned tables. Attempts to replicate other types of relations, such as views, materialized views, or foreign tables, will result in an error. </para> + <para> + Replication is not supported for foreign tables. When used as partitions + of partitioned tables, publishing of the partitioned table is only allowed + if the <literal>publish_via_partition_root</literal> is set to + <literal>false</literal>. In this mode, changes to a partition that is a + foreign table are ignored for the purposes of replication, and data + contained in them is not included during initial synchronization. + </para> </listitem> <listitem> diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml index 57dec28a5df..3d8992e27e3 100644 --- a/doc/src/sgml/ref/create_subscription.sgml +++ b/doc/src/sgml/ref/create_subscription.sgml @@ -545,6 +545,13 @@ WHERE C.relnamespace = N.oid AND PT.pubname IN (<pub-names>); </programlisting></para> + <para> + When using a subscription parameter copy_data = true, corresponding + publications are checked if it has publish_via_partition_root = true and + has partitioned table with foreign table as partition. If this scenario is + detected we ERROR is logged to the user. + </para> + </refsect1> <refsect1> diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c index d6f94db5d99..56255dcd5cc 100644 --- a/src/backend/catalog/pg_publication.c +++ b/src/backend/catalog/pg_publication.c @@ -304,7 +304,7 @@ check_and_fetch_column_list(Publication *pub, Oid relid, MemoryContext mcxt, /* * Gets the relations based on the publication partition option for a specified - * relation. + * relation. Foreign tables are not included. */ List * GetPubPartitionOptionRelations(List *result, PublicationPartOpt pub_partopt, @@ -313,25 +313,21 @@ GetPubPartitionOptionRelations(List *result, PublicationPartOpt pub_partopt, if (get_rel_relkind(relid) == RELKIND_PARTITIONED_TABLE && pub_partopt != PUBLICATION_PART_ROOT) { - List *all_parts = find_all_inheritors(relid, NoLock, - NULL); + List *all_parts = find_all_inheritors(relid, NoLock, NULL); - if (pub_partopt == PUBLICATION_PART_ALL) - result = list_concat(result, all_parts); - else if (pub_partopt == PUBLICATION_PART_LEAF) + foreach_oid(partOid, all_parts) { - ListCell *lc; + char relkind = get_rel_relkind(partOid); - foreach(lc, all_parts) - { - Oid partOid = lfirst_oid(lc); + if (relkind == RELKIND_FOREIGN_TABLE) + continue; - if (get_rel_relkind(partOid) != RELKIND_PARTITIONED_TABLE) - result = lappend_oid(result, partOid); - } + if (pub_partopt == PUBLICATION_PART_LEAF && + relkind == RELKIND_PARTITIONED_TABLE) + continue; + + result = lappend_oid(result, partOid); } - else - Assert(false); } else result = lappend_oid(result, relid); diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c index 4aec73bcc6b..a81b334d127 100644 --- a/src/backend/commands/subscriptioncmds.c +++ b/src/backend/commands/subscriptioncmds.c @@ -112,6 +112,9 @@ static List *merge_publications(List *oldpublist, List *newpublist, bool addpub, static void ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err); static void CheckAlterSubOption(Subscription *sub, const char *option, bool slot_needs_update, bool isTopLevel); +static void check_publications_foreign_parts(WalReceiverConn *wrconn, + List *publications, bool copydata, + char *subname); /* @@ -723,6 +726,8 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, check_publications(wrconn, publications); check_publications_origin(wrconn, publications, opts.copy_data, opts.origin, NULL, 0, stmt->subname); + check_publications_foreign_parts(wrconn, publications, + opts.copy_data, stmt->subname); /* * Set sync state based on if we were asked to do data copy or @@ -883,6 +888,8 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data, check_publications_origin(wrconn, sub->publications, copy_data, sub->origin, subrel_local_oids, subrel_count, sub->name); + check_publications_foreign_parts(wrconn, sub->publications, copy_data, + sub->name); /* * Rels that we want to remove from subscription and drop any slots @@ -2507,3 +2514,97 @@ defGetStreamingMode(DefElem *def) def->defname))); return LOGICALREP_STREAM_OFF; /* keep compiler quiet */ } + +/* + * check_publications_foreign_parts + * Check if the publications, on which subscriber is subscribing, publishes any + * partitioned table that has an foreign table as its partition and has + * publish_via_partition_root set as true. The check is performed only if + * copy_data is set as true for the subscription. + * + * DML data changes are not published for data in foreign tables, + * and yet the tablesync worker is not smart enough to omit data from + * foreign tables when they are partitions of partitioned tables. To + * avoid the inconsistencies that would result, we disallow foreign + * tables from being published generally. However, it's possible for + * partitioned tables to have foreign tables as partitions, and we would + * like to allow publishing those partitioned tables so that the other + * partitions are replicated. + * + * This function is in charge of detecting if publisher with + * publish_via_partition_root=true publishes a partitioned table that has a + * foreign table as a partition. + * + * When publish_via_partition_root is false, each partition published for + * replication is listed individually in pg_subscription_rel, and we + * don't add partitions that are foreign tables, so this check is not + * needed. + */ +static void +check_publications_foreign_parts(WalReceiverConn *wrconn, List *publications, + bool copydata, char *subname) +{ + WalRcvExecResult *res; + StringInfoData cmd; + TupleTableSlot *slot; + Oid tableRow[1] = {TEXTOID}; + List *publist = NIL; + int i; + + if (!copydata) + return; + + initStringInfo(&cmd); + appendStringInfoString(&cmd, + "SELECT DISTINCT P.pubname AS pubname " + "from pg_catalog.pg_publication p, LATERAL " + "pg_get_publication_tables(p.pubname) gpt, LATERAL " + "pg_partition_tree(gpt.relid) gt JOIN pg_catalog.pg_foreign_table ft ON " + "ft.ftrelid = gt.relid WHERE p.pubviaroot = true AND p.pubname IN ("); + + GetPublicationsStr(publications, &cmd, true); + appendStringInfoString(&cmd, ")\n"); + + res = walrcv_exec(wrconn, cmd.data, 1, tableRow); + pfree(cmd.data); + + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + (errcode(ERRCODE_CONNECTION_FAILURE), + errmsg("could not receive list of replicated tables from the publisher: %s", + res->err))); + + /* Process tables. */ + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); + while (tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + { + char *pubname; + bool isnull; + + pubname = TextDatumGetCString(slot_getattr(slot, 1, &isnull)); + Assert(!isnull); + + ExecClearTuple(slot); + publist = list_append_unique(publist, makeString(pubname)); + } + + if (publist) + { + StringInfo pubnames = makeStringInfo(); + + /* Prepare the list of publication(s) for warning message. */ + GetPublicationsStr(publist, pubnames, false); + ereport(ERROR, + errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("subscription \"%s\" with copy_data = true cannot subscribe to a publication with publish_via_partition_root = true and publishes partitioned table with foreign table as partition", + subname), + errdetail_plural("The subscription being created on a publication (%s) with publish_via_root_partition = true and contains partitioned tables with foreign table as partition ", + "The subscription being created on publications (%s) with publish_via_root_partition = true and contains partitioned tables with foreign table as partition ", + list_length(publist), pubnames->data), + errhint("Drop the foreign table from the publication or set publish_via_partition_root = false on publication or set copy_data = false.")); + } + + ExecDropSingleTupleTableSlot(slot); + + walrcv_clear_result(res); +} diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 54ad38247aa..345e0e8b279 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -1133,6 +1133,52 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, errmsg("\"%s\" is not partitioned", RelationGetRelationName(parent)))); + /* + * If we're creating a partition that's a foreign table, verify that + * the parent table is not in a publication with + * publish_via_partition_root enabled. + */ + if (rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE) + { + Oid schemaid; + List *puboids; + List *ancestors; + + /* Start with publications of all tables */ + puboids = GetAllTablesPublications(); + + /* capture all publications that include this relation directly */ + puboids = list_concat(puboids, GetRelationPublications(parent->rd_id)); + schemaid = RelationGetNamespace(parent); + puboids = list_concat(puboids, GetSchemaPublications(schemaid)); + + /* and do the same for its ancestors, if any */ + ancestors = get_partition_ancestors(parent->rd_id); + foreach_oid(ancestor, ancestors) + { + puboids = list_concat(puboids, GetRelationPublications(ancestor)); + schemaid = get_rel_namespace(ancestor); + puboids = list_concat(puboids, GetSchemaPublications(schemaid)); + } + + /* Check the publish_via_partition_root bit for each of those */ + list_sort(puboids, list_oid_cmp); + list_deduplicate_oid(puboids); + foreach_oid(puboid, puboids) + { + Publication *pub = GetPublication(puboid); + + if (pub->pubviaroot) + ereport(ERROR, + errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot create foreign table \"%s\" as a partition of \"%s\"", + RelationGetRelationName(rel), RelationGetRelationName(parent)), + errdetail("Partitioned table \"%s\" is published with option \"%s\" in publication \"%s\".", + RelationGetRelationName(parent), + "publish_via_partition_root", pub->name)); + } + } + /* * The partition constraint of the default partition depends on the * partition bounds of every other partition. It is possible that @@ -20291,6 +20337,70 @@ ATExecAttachPartition(List **wqueue, Relation rel, PartitionCmd *cmd, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot attach temporary relation of another session as partition"))); + /* + * If the relation to attach is a foreign table, or a partitioned table + * that contains a foreign table as partition, then verify that the parent + * table is not in a publication with publish_via_partition_root enabled. + */ + if (attachrel->rd_rel->relkind == RELKIND_FOREIGN_TABLE || + (attachrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE && + RelationHasForeignPartition(attachrel))) + { + Oid schemaid; + List *puboids; + List *ancestors; + + /* Start with publications of all tables */ + puboids = GetAllTablesPublications(); + + /* capture all publications that include this relation directly */ + puboids = list_concat(puboids, GetRelationPublications(rel->rd_id)); + schemaid = RelationGetNamespace(rel); + puboids = list_concat(puboids, GetSchemaPublications(schemaid)); + + /* and do the same for its ancestors, if any */ + ancestors = get_partition_ancestors(rel->rd_id); + foreach_oid(ancestor, ancestors) + { + puboids = list_concat(puboids, GetRelationPublications(ancestor)); + schemaid = get_rel_namespace(ancestor); + puboids = list_concat(puboids, GetSchemaPublications(schemaid)); + } + + /* Now check the publish_via_partition_root bit for each of those */ + list_sort(puboids, list_oid_cmp); + list_deduplicate_oid(puboids); + foreach_oid(puboid, puboids) + { + Publication *pub; + + pub = GetPublication(puboid); + if (pub->pubviaroot) + { + if (attachrel->rd_rel->relkind == RELKIND_FOREIGN_TABLE) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot attach foreign table \"%s\" to partition table \"%s\"", + RelationGetRelationName(attachrel), + RelationGetRelationName(rel)), + errdetail("Partitioned table \"%s\" is published with option \"%s\" in publication \"%s\".", + RelationGetRelationName(rel), + "publish_via_partition_root", + pub->name))); + else + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot attach table \"%s\" with a partition that's a foreign table to partition table \"%s\"", + RelationGetRelationName(attachrel), + RelationGetRelationName(rel)), + errdetail("Partitioned table \"%s\" is published with option \"%s\" in publication \"%s\".", + RelationGetRelationName(rel), + "publish_via_partition_root", + pub->name))); + } + } + } + /* * Check if attachrel has any identity columns or any columns that aren't * in the parent. diff --git a/src/backend/partitioning/partdesc.c b/src/backend/partitioning/partdesc.c index 328b4d450e4..56d176625e6 100644 --- a/src/backend/partitioning/partdesc.c +++ b/src/backend/partitioning/partdesc.c @@ -506,3 +506,35 @@ get_default_oid_from_partdesc(PartitionDesc partdesc) return InvalidOid; } + +/* + * Return true if the given partitioned table ultimately contains a + * partition that is a foreign table, false otherwise. + */ +bool +RelationHasForeignPartition(Relation rel) +{ + PartitionDesc pd = RelationGetPartitionDesc(rel, true); + + for (int i = 0; i < pd->nparts; i++) + { + if (pd->is_leaf[i]) + { + if (get_rel_relkind(pd->oids[i]) == RELKIND_FOREIGN_TABLE) + return true; + } + else + { + Relation part; + bool ret; + + part = table_open(pd->oids[i], NoLock); + ret = RelationHasForeignPartition(part); + table_close(part, NoLock); + if (ret) + return true; + } + } + + return false; +} diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index 940fc77fc2e..2cf407a3169 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -31,6 +31,7 @@ static void check_new_cluster_logical_replication_slots(void); static void check_new_cluster_subscription_configuration(void); static void check_old_cluster_for_valid_slots(void); static void check_old_cluster_subscription_state(void); +static void check_for_valid_publication(ClusterInfo *cluster); /* * DataTypesUsageChecks - definitions of data type checks for the old cluster @@ -633,6 +634,8 @@ check_and_dump_old_cluster(void) check_old_cluster_subscription_state(); } + check_for_valid_publication(&old_cluster); + check_for_data_types_usage(&old_cluster); /* @@ -2272,3 +2275,89 @@ check_old_cluster_subscription_state(void) else check_ok(); } + +/* + * Callback function for processing results of query for + * check_for_valid_publication()'s UpgradeTask. If the query returned + * any rows (i.e., the check failed), write the details to the report file. + */ +static void +process_pub_check(DbInfo *dbinfo, PGresult *res, void *arg) +{ + UpgradeTaskReport *report = (UpgradeTaskReport *) arg; + int ntups = PQntuples(res); + int i_puboid = PQfnumber(res, "oid"); + int i_pubviaroot = PQfnumber(res, "pubviaroot"); + int i_puballtables = PQfnumber(res, "puballtables"); + int i_ftrelid = PQfnumber(res, "ftrelid"); + + if (ntups == 0) + return; + + if (report->file == NULL && + (report->file = fopen_priv(report->path, "w")) == NULL) + pg_fatal("could not open file \"%s\": %m", report->path); + + fprintf(report->file, "In database: %s\n", dbinfo->db_name); + + for (int rowno = 0; rowno < ntups; rowno++) + fprintf(report->file, "The publication with oid: %s, has foreign table with oid: %s as partition of a published table. " + "Publication is published with publish_via_partition_root = %s and it is published %s.\n", + PQgetvalue(res, rowno, i_puboid), + PQgetvalue(res, rowno, i_ftrelid), + strcmp(PQgetvalue(res, rowno, i_pubviaroot), "f") ? "true" : "false", + strcmp(PQgetvalue(res, rowno, i_puballtables), "f") ? "FOR ALL TABLES" : "FOR TABLE/FOR TABLES IN SCHEMA"); +} + +/* + * check_for_valid_publication + * + * Verify if the publication has any published foreign table which are + * partitions of a table. + * + * For PG 18 and below foreign tables can be indirectly published when it is + * partition of a partitioned table. Avoid upgrading the cluster if it has a + * publication with publish_via_partition_root = true and has foreign table + * which is partition of a published table. + */ +static void +check_for_valid_publication(ClusterInfo *cluster) +{ + UpgradeTaskReport report; + UpgradeTask *task; + const char *query; + + if (GET_MAJOR_VERSION(cluster->major_version) >= 1900) + return; + + prep_status("Checking for invalid publications"); + + report.file = NULL; + snprintf(report.path, sizeof(report.path), "%s/%s", + log_opts.basedir, + "pubs_invalid.txt"); + + query = "SELECT p.oid, p.pubviaroot, p.puballtables, ft.ftrelid " + "from pg_catalog.pg_publication p, LATERAL " + "pg_get_publication_tables(p.pubname) gpt, LATERAL " + "pg_partition_tree(gpt.relid) gt JOIN pg_catalog.pg_foreign_table ft ON " + "ft.ftrelid = gt.relid WHERE p.pubviaroot = true;"; + + task = upgrade_task_create(); + upgrade_task_add_step(task, query, + process_pub_check, + true, &report); + upgrade_task_run(task, cluster); + upgrade_task_free(task); + + if (report.file) + { + fclose(report.file); + pg_log(PG_REPORT, "fatal"); + pg_fatal("Your installation contains publications, which has foreign table which are partitions of published table.\n" + "A list of potentially-affected publications is in the file:\n" + " %s", report.path); + } + else + check_ok(); +} diff --git a/src/include/partitioning/partdesc.h b/src/include/partitioning/partdesc.h index 34533f7004c..5fbafdc06f9 100644 --- a/src/include/partitioning/partdesc.h +++ b/src/include/partitioning/partdesc.h @@ -71,5 +71,6 @@ extern PartitionDesc PartitionDirectoryLookup(PartitionDirectory, Relation); extern void DestroyPartitionDirectory(PartitionDirectory pdir); extern Oid get_default_oid_from_partdesc(PartitionDesc partdesc); +extern bool RelationHasForeignPartition(Relation rel); #endif /* PARTDESC_H */ diff --git a/src/test/subscription/Makefile b/src/test/subscription/Makefile index 50b65d8f6ea..11041f79667 100644 --- a/src/test/subscription/Makefile +++ b/src/test/subscription/Makefile @@ -14,6 +14,7 @@ top_builddir = ../../.. include $(top_builddir)/src/Makefile.global EXTRA_INSTALL = contrib/hstore +EXTRA_INSTALL += contrib/postgres_fdw export with_icu diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl index 61b0cb4aa1a..41cfdc86580 100644 --- a/src/test/subscription/t/013_partition.pl +++ b/src/test/subscription/t/013_partition.pl @@ -8,6 +8,9 @@ use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; +my $stdout; +my $stderr; + # setup my $node_publisher = PostgreSQL::Test::Cluster->new('publisher'); @@ -891,4 +894,140 @@ $result = $node_subscriber2->safe_psql('postgres', "SELECT a, b, c FROM tab5_1 ORDER BY 1"); is($result, qq(4||1), 'updates of tab5 replicated correctly'); +# Test with partitioned table with foreign table as its partition. +$node_publisher->safe_psql('postgres', "CREATE DATABASE db1"); +$node_publisher->safe_psql( + 'db1', + q{CREATE EXTENSION IF NOT EXISTS postgres_fdw; + CREATE SERVER fdw FOREIGN DATA WRAPPER postgres_fdw; + CREATE USER MAPPING FOR CURRENT_USER SERVER fdw;}); + +$node_publisher->safe_psql( + 'db1', + q{CREATE SCHEMA sc1; + CREATE SCHEMA sc2; + CREATE TABLE sc1.tab6(id int) PARTITION BY RANGE(id); + CREATE TABLE sc1.tab6_1 PARTITION OF sc1.tab6 FOR VALUES FROM (0) TO (5); + CREATE TABLE sc1.tab6_2 PARTITION OF sc1.tab6 FOR VALUES FROM (5) TO (15) PARTITION BY RANGE(id); + CREATE FOREIGN TABLE sc1.tab6_2_1 PARTITION OF sc1.tab6_2 FOR VALUES FROM (10) TO (15) SERVER fdw;} +); +$node_publisher->safe_psql( + 'db1', + q{CREATE PUBLICATION pub_foreign_1 FOR TABLE sc1.tab6 with (publish_via_partition_root); + CREATE PUBLICATION pub_foreign_2 FOR TABLES IN SCHEMA sc1 with (publish_via_partition_root); + CREATE PUBLICATION pub_foreign_3 FOR ALL TABLES with (publish_via_partition_root); + CREATE PUBLICATION pub_foreign_4 FOR TABLES IN SCHEMA sc2 with (publish_via_partition_root); + }); + +$node_subscriber1->safe_psql('postgres', "CREATE DATABASE db1"); +$node_subscriber1->safe_psql( + 'db1', + q{CREATE SCHEMA sc1; + CREATE SCHEMA sc2; + CREATE TABLE sc1.tab6(id int) PARTITION BY RANGE(id); + CREATE TABLE sc1.tab6_1 PARTITION OF sc1.tab6 FOR VALUES FROM (0) TO (5); + CREATE TABLE sc1.tab6_2 PARTITION OF sc1.tab6 FOR VALUES FROM (5) TO (15) PARTITION BY RANGE(id); + CREATE TABLE sc1.tab6_2_1 PARTITION OF sc1.tab6_2 FOR VALUES FROM (10) TO (15);} +); + +$publisher_connstr = $node_publisher->connstr . ' dbname=db1'; + +# Create subscription on publication which has partitioned table with foreign +# table as its partition. +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "CREATE SUBSCRIPTION sub_foreign_1 CONNECTION '$publisher_connstr' PUBLICATION pub_foreign_1;" +); +like( + $stderr, + qr/ERROR: subscription "sub_foreign_1" with copy_data = true cannot subscribe to a publication with publish_via_partition_root = true and publishes partitioned table with foreign table as partition/, + "Create subscription on a publication FOR TABLE on partitioned table with foreign table as its partition" +); + +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "CREATE SUBSCRIPTION sub_foreign_1 CONNECTION '$publisher_connstr' PUBLICATION pub_foreign_2;" +); +like( + $stderr, + qr/ERROR: subscription "sub_foreign_1" with copy_data = true cannot subscribe to a publication with publish_via_partition_root = true and publishes partitioned table with foreign table as partition/, + "Create subscription on a publication FOR TABLES IN SCHEMA on partitioned table with foreign table as its partition" +); +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "CREATE SUBSCRIPTION sub_foreign_1 CONNECTION '$publisher_connstr' PUBLICATION pub_foreign_3;" +); +like( + $stderr, + qr/ERROR: subscription "sub_foreign_1" with copy_data = true cannot subscribe to a publication with publish_via_partition_root = true and publishes partitioned table with foreign table as partition/, + "Create subscription on a publication FOR ALL TABLES on partitioned table with foreign table as its partition" +); + +# If publication is created with publish_via_partition_root = false, we can +# create subscription on it. But replication of partitioned table with foreign +# is skipped +$node_publisher->safe_psql('db1', + "ALTER PUBLICATION pub_foreign_1 SET (publish_via_partition_root = false)" +); +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "CREATE SUBSCRIPTION sub_foreign_1 CONNECTION '$publisher_connstr' PUBLICATION pub_foreign_1;" +); +$node_subscriber1->safe_psql('db1', 'DROP SUBSCRIPTION sub_foreign_1'); + +# if subscription is created with copy_data=false, we can create subscription +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "CREATE SUBSCRIPTION sub_foreign_1 CONNECTION '$publisher_connstr' PUBLICATION pub_foreign_2 WITH (copy_data=false)" +); +$node_subscriber1->safe_psql('db1', 'DROP SUBSCRIPTION sub_foreign_1'); + +# Alter subscription to include publisher which has partitioned table with +# foreign table as its partition. +$node_subscriber1->safe_psql('db1', + "CREATE SUBSCRIPTION sub_foreign_1 CONNECTION '$publisher_connstr' PUBLICATION pub_foreign_4;" +); +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "ALTER SUBSCRIPTION sub_foreign_1 ADD PUBLICATION pub_foreign_2;"); +like( + $stderr, + qr/ERROR: subscription "sub_foreign_1" with copy_data = true cannot subscribe to a publication with publish_via_partition_root = true and publishes partitioned table with foreign table as partition/, + "Add publication which has partitioned table with foreign table as its partition to subscription" +); +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "ALTER SUBSCRIPTION sub_foreign_1 SET PUBLICATION pub_foreign_2;"); +like( + $stderr, + qr/ERROR: subscription "sub_foreign_1" with copy_data = true cannot subscribe to a publication with publish_via_partition_root = true and publishes partitioned table with foreign table as partition/, + "set publication which has partitioned table with foreign table as its partition for subscription" +); + +# Create foreign table as a partition of partitioned table already part of +# publication. +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "CREATE FOREIGN TABLE sc1.tab6_2_1 PARTITION OF sc1.tab6_2 FOR VALUES FROM (10) TO (15) SERVER fdw;" +); +$node_publisher->safe_psql('db1', "DROP FOREIGN TABLE sc1.tab6_2_1"); +($result, $stdout, $stderr) = $node_publisher->psql('db1', + "CREATE FOREIGN TABLE sc1.tab6_2_1 PARTITION OF sc1.tab6_2 FOR VALUES FROM (10) TO (15) SERVER fdw" +); +like( + $stderr, + qr/ERROR: cannot create foreign table "tab6_2_1" as a partition of "tab6_2"/, + "Create foreign table as a partition of partitioned table already part of publication" +); + +# Attach foreign table as a partition of partitioned table already part of +# publication. +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "CREATE FOREIGN TABLE sc1.tab6_2_1 (id int) SERVER fdw"); +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "ALTER TABLE sc1.tab6_2 ATTACH PARTITION sc1.tab6_2_1 FOR VALUES FROM (10) TO (15)" +); +$node_publisher->safe_psql('db1', + "CREATE FOREIGN TABLE sc1.tab6_2_1 (id int) SERVER fdw"); +($result, $stdout, $stderr) = $node_publisher->psql('db1', + "ALTER TABLE sc1.tab6_2 ATTACH PARTITION sc1.tab6_2_1 FOR VALUES FROM (10) TO (15)" +); +like( + $stderr, + qr/ERROR: cannot attach foreign table "tab6_2_1" to partition table "tab6_2"/, + "Attach foreign table as a partition of already published partitioned table" +); + done_testing(); -- 2.34.1 ^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: Restrict publishing of partitioned table with a foreign table as partition 2025-04-28 12:48 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-04-28 14:27 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> 2025-05-09 14:55 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-05-10 17:59 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> 2025-05-15 12:48 ` Re: Restrict publishing of partitioned table with a foreign table as partition Amit Kapila <[email protected]> 2025-05-19 16:32 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> @ 2025-06-04 10:41 ` Ajin Cherian <[email protected]> 2025-06-09 05:57 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 0 siblings, 1 reply; 19+ messages in thread From: Ajin Cherian @ 2025-06-04 10:41 UTC (permalink / raw) To: Shlok Kyal <[email protected]>; +Cc: Amit Kapila <[email protected]>; Álvaro Herrera <[email protected]>; vignesh C <[email protected]>; Sergey Tatarintsev <[email protected]>; [email protected] On Tue, May 20, 2025 at 2:33 AM Shlok Kyal <[email protected]> wrote: > > This approach seems better to me. I have created a patch with the > above approach. > > Thanks and Regards, > Shlok Kyal Some quick comments on the patch: 1. In doc/src/sgml/ref/create_subscription.sgml: + has partitioned table with foreign table as partition. If this scenario is + detected we ERROR is logged to the user. + </para> + Should be: "If this scenario is detected an ERROR is logged to the user." (remove "we"). In src/backend/commands/subscriptioncmds.c: 2. The comment header: + * This function is in charge of detecting if publisher with + * publish_via_partition_root=true publishes a partitioned table that has a + * foreign table as a partition. Add "and throw an error if found" at the end of that sentence to correctly describe what the function does. 3. + appendStringInfoString(&cmd, + "SELECT DISTINCT P.pubname AS pubname " + "from pg_catalog.pg_publication p, LATERAL " + "pg_get_publication_tables(p.pubname) gpt, LATERAL " + "pg_partition_tree(gpt.relid) gt JOIN pg_catalog.pg_foreign_table ft ON " + "ft.ftrelid = gt.relid WHERE p.pubviaroot = true AND p.pubname IN ("); use FROM rather than from to maintain SQL style consistency. 4. + errdetail_plural("The subscription being created on a publication (%s) with publish_via_root_partition = true and contains partitioned tables with foreign table as partition ", + "The subscription being created on publications (%s) with publish_via_root_partition = true and contains partitioned tables with foreign table as partition ", + list_length(publist), pubnames->data), I think you meant "publish_via_partition_root" here and not "publish_via_root_partition ". regards, Ajin Cherian Fujitsu Australia ^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: Restrict publishing of partitioned table with a foreign table as partition 2025-04-28 12:48 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-04-28 14:27 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> 2025-05-09 14:55 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-05-10 17:59 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> 2025-05-15 12:48 ` Re: Restrict publishing of partitioned table with a foreign table as partition Amit Kapila <[email protected]> 2025-05-19 16:32 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-06-04 10:41 ` Re: Restrict publishing of partitioned table with a foreign table as partition Ajin Cherian <[email protected]> @ 2025-06-09 05:57 ` Shlok Kyal <[email protected]> 2025-07-01 02:50 ` Re: Restrict publishing of partitioned table with a foreign table as partition Ajin Cherian <[email protected]> 0 siblings, 1 reply; 19+ messages in thread From: Shlok Kyal @ 2025-06-09 05:57 UTC (permalink / raw) To: Ajin Cherian <[email protected]>; +Cc: Amit Kapila <[email protected]>; Álvaro Herrera <[email protected]>; vignesh C <[email protected]>; Sergey Tatarintsev <[email protected]>; [email protected] On Wed, 4 Jun 2025 at 16:12, Ajin Cherian <[email protected]> wrote: > > On Tue, May 20, 2025 at 2:33 AM Shlok Kyal <[email protected]> wrote: > > > > This approach seems better to me. I have created a patch with the > > above approach. > > > > Thanks and Regards, > > Shlok Kyal > > Some quick comments on the patch: > 1. In doc/src/sgml/ref/create_subscription.sgml: > + has partitioned table with foreign table as partition. If this scenario is > + detected we ERROR is logged to the user. > + </para> > + > > Should be: "If this scenario is detected an ERROR is logged to the > user." (remove "we"). > > In src/backend/commands/subscriptioncmds.c: > 2. The comment header: > + * This function is in charge of detecting if publisher with > + * publish_via_partition_root=true publishes a partitioned table that has a > + * foreign table as a partition. > > Add "and throw an error if found" at the end of that sentence to > correctly describe what the function does. > > 3. > + appendStringInfoString(&cmd, > + "SELECT DISTINCT P.pubname AS pubname " > + "from pg_catalog.pg_publication p, LATERAL " > + "pg_get_publication_tables(p.pubname) gpt, LATERAL " > + "pg_partition_tree(gpt.relid) gt JOIN > pg_catalog.pg_foreign_table ft ON " > + "ft.ftrelid = gt.relid WHERE p.pubviaroot > = true AND p.pubname IN ("); > > use FROM rather than from to maintain SQL style consistency. > > 4. > + errdetail_plural("The subscription being created on a > publication (%s) with publish_via_root_partition = true and contains > partitioned tables with foreign table as partition ", > + "The subscription being created on > publications (%s) with publish_via_root_partition = true and contains > partitioned tables with foreign table as partition ", > + list_length(publist), pubnames->data), > > I think you meant "publish_via_partition_root" here and not > "publish_via_root_partition ". > I have addressed all the comments and attached the updated patch. Thanks and Regards, Shlok Kyal Attachments: [application/octet-stream] v16-0001-Restrict-publishing-of-partitioned-table-with-fo.patch (29.2K, ../../CANhcyEVU9eqKtnL3vnNJ_fN7q5PwUwv2ceT8p7YaYKHedqLrpQ@mail.gmail.com/2-v16-0001-Restrict-publishing-of-partitioned-table-with-fo.patch) download | inline diff: From e3dad3aa33ae8116b9b62ae4c11a9e8a143d8cca Mon Sep 17 00:00:00 2001 From: Shlok Kyal <[email protected]> Date: Mon, 19 May 2025 19:47:04 +0530 Subject: [PATCH v16] Restrict publishing of partitioned table with foreign table as its partition Logical replication of foreign table is not supported and we throw an error in this case. But when we create a publication on a partitioned table that has a foreign table as its partition, the initial sync of such table is successful and we should avoid such cases. Current Behaviour in HEAD, when publication is created: 1. with publish_via_partition_root = true The root table is published, and initial data from partitions that are foreign tables is replicated. 2. with publish_via_partition_root = false and FOR ALL TABLES All leaf tables except partitions that are foreign tables are published. 3. with publish_via_partition_root = false and FOR TABLE/ FOR TABLES IN SCHEMA All leaf tables are published, including initial data from partitions that are foreign tables. With this patch we have following behaviour: 1. with publish_via_partition_root = true An error is thrown when we try to create subscription (with copy_data = true) on publication with partitioned table with foreign partition. When copy_data = false, we donot need this check as initial sync is not performed. Also we restrict creating/attaching a foreign table as partition of a partitioned table which is part of any publication. 2. with publish_via_partition_root = false We skip publishing partitions that are foreign tables. This is done by avoid adding such partitions in pg_subscription_rel catalog table. We have introduced two functions 'RelationHasForeignPartition' and 'check_publications_foreign_parts'. In 'RelationHasForeignPartition' we go through the child nodes of a partition and check if it has a foreign table. In 'check_publications_foreign_parts' we check if a subscription with copy_data = true is subscribing to a publication which has partitioned table with foreign table as its partition. --- doc/src/sgml/logical-replication.sgml | 10 +- doc/src/sgml/ref/create_subscription.sgml | 7 ++ src/backend/catalog/pg_publication.c | 26 ++-- src/backend/commands/subscriptioncmds.c | 101 ++++++++++++++++ src/backend/commands/tablecmds.c | 110 +++++++++++++++++ src/backend/partitioning/partdesc.c | 32 +++++ src/bin/pg_upgrade/check.c | 89 ++++++++++++++ src/include/partitioning/partdesc.h | 1 + src/test/subscription/Makefile | 1 + src/test/subscription/t/013_partition.pl | 139 ++++++++++++++++++++++ 10 files changed, 500 insertions(+), 16 deletions(-) diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index 686dd441d02..460aa929b73 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -2079,10 +2079,18 @@ CONTEXT: processing remote data for replication origin "pg_16395" during "INSER <listitem> <para> - Replication is only supported by tables, including partitioned tables. + Replication is only supported for tables, including partitioned tables. Attempts to replicate other types of relations, such as views, materialized views, or foreign tables, will result in an error. </para> + <para> + Replication is not supported for foreign tables. When used as partitions + of partitioned tables, publishing of the partitioned table is only allowed + if the <literal>publish_via_partition_root</literal> is set to + <literal>false</literal>. In this mode, changes to a partition that is a + foreign table are ignored for the purposes of replication, and data + contained in them is not included during initial synchronization. + </para> </listitem> <listitem> diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml index 57dec28a5df..8963123725b 100644 --- a/doc/src/sgml/ref/create_subscription.sgml +++ b/doc/src/sgml/ref/create_subscription.sgml @@ -545,6 +545,13 @@ WHERE C.relnamespace = N.oid AND PT.pubname IN (<pub-names>); </programlisting></para> + <para> + When using a subscription parameter copy_data = true, corresponding + publications are checked if it has publish_via_partition_root = true and + has partitioned table with foreign table as partition. If this scenario is + detected an ERROR is logged to the user. + </para> + </refsect1> <refsect1> diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c index d6f94db5d99..56255dcd5cc 100644 --- a/src/backend/catalog/pg_publication.c +++ b/src/backend/catalog/pg_publication.c @@ -304,7 +304,7 @@ check_and_fetch_column_list(Publication *pub, Oid relid, MemoryContext mcxt, /* * Gets the relations based on the publication partition option for a specified - * relation. + * relation. Foreign tables are not included. */ List * GetPubPartitionOptionRelations(List *result, PublicationPartOpt pub_partopt, @@ -313,25 +313,21 @@ GetPubPartitionOptionRelations(List *result, PublicationPartOpt pub_partopt, if (get_rel_relkind(relid) == RELKIND_PARTITIONED_TABLE && pub_partopt != PUBLICATION_PART_ROOT) { - List *all_parts = find_all_inheritors(relid, NoLock, - NULL); + List *all_parts = find_all_inheritors(relid, NoLock, NULL); - if (pub_partopt == PUBLICATION_PART_ALL) - result = list_concat(result, all_parts); - else if (pub_partopt == PUBLICATION_PART_LEAF) + foreach_oid(partOid, all_parts) { - ListCell *lc; + char relkind = get_rel_relkind(partOid); - foreach(lc, all_parts) - { - Oid partOid = lfirst_oid(lc); + if (relkind == RELKIND_FOREIGN_TABLE) + continue; - if (get_rel_relkind(partOid) != RELKIND_PARTITIONED_TABLE) - result = lappend_oid(result, partOid); - } + if (pub_partopt == PUBLICATION_PART_LEAF && + relkind == RELKIND_PARTITIONED_TABLE) + continue; + + result = lappend_oid(result, partOid); } - else - Assert(false); } else result = lappend_oid(result, relid); diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c index 4aec73bcc6b..2557b06313d 100644 --- a/src/backend/commands/subscriptioncmds.c +++ b/src/backend/commands/subscriptioncmds.c @@ -112,6 +112,9 @@ static List *merge_publications(List *oldpublist, List *newpublist, bool addpub, static void ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err); static void CheckAlterSubOption(Subscription *sub, const char *option, bool slot_needs_update, bool isTopLevel); +static void check_publications_foreign_parts(WalReceiverConn *wrconn, + List *publications, bool copydata, + char *subname); /* @@ -723,6 +726,8 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, check_publications(wrconn, publications); check_publications_origin(wrconn, publications, opts.copy_data, opts.origin, NULL, 0, stmt->subname); + check_publications_foreign_parts(wrconn, publications, + opts.copy_data, stmt->subname); /* * Set sync state based on if we were asked to do data copy or @@ -883,6 +888,8 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data, check_publications_origin(wrconn, sub->publications, copy_data, sub->origin, subrel_local_oids, subrel_count, sub->name); + check_publications_foreign_parts(wrconn, sub->publications, copy_data, + sub->name); /* * Rels that we want to remove from subscription and drop any slots @@ -2507,3 +2514,97 @@ defGetStreamingMode(DefElem *def) def->defname))); return LOGICALREP_STREAM_OFF; /* keep compiler quiet */ } + +/* + * check_publications_foreign_parts + * Check if the publications, on which subscriber is subscribing, publishes any + * partitioned table that has an foreign table as its partition and has + * publish_via_partition_root set as true. The check is performed only if + * copy_data is set as true for the subscription. + * + * DML data changes are not published for data in foreign tables, + * and yet the tablesync worker is not smart enough to omit data from + * foreign tables when they are partitions of partitioned tables. To + * avoid the inconsistencies that would result, we disallow foreign + * tables from being published generally. However, it's possible for + * partitioned tables to have foreign tables as partitions, and we would + * like to allow publishing those partitioned tables so that the other + * partitions are replicated. + * + * This function is in charge of detecting if publisher with + * publish_via_partition_root=true publishes a partitioned table that has a + * foreign table as a partition and throw an error if found. + * + * When publish_via_partition_root is false, each partition published for + * replication is listed individually in pg_subscription_rel, and we + * don't add partitions that are foreign tables, so this check is not + * needed. + */ +static void +check_publications_foreign_parts(WalReceiverConn *wrconn, List *publications, + bool copydata, char *subname) +{ + WalRcvExecResult *res; + StringInfoData cmd; + TupleTableSlot *slot; + Oid tableRow[1] = {TEXTOID}; + List *publist = NIL; + int i; + + if (!copydata) + return; + + initStringInfo(&cmd); + appendStringInfoString(&cmd, + "SELECT DISTINCT P.pubname AS pubname " + "FROM pg_catalog.pg_publication p, LATERAL " + "pg_get_publication_tables(p.pubname) gpt, LATERAL " + "pg_partition_tree(gpt.relid) gt JOIN pg_catalog.pg_foreign_table ft ON " + "ft.ftrelid = gt.relid WHERE p.pubviaroot = true AND p.pubname IN ("); + + GetPublicationsStr(publications, &cmd, true); + appendStringInfoString(&cmd, ")\n"); + + res = walrcv_exec(wrconn, cmd.data, 1, tableRow); + pfree(cmd.data); + + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + (errcode(ERRCODE_CONNECTION_FAILURE), + errmsg("could not receive list of replicated tables from the publisher: %s", + res->err))); + + /* Process tables. */ + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); + while (tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + { + char *pubname; + bool isnull; + + pubname = TextDatumGetCString(slot_getattr(slot, 1, &isnull)); + Assert(!isnull); + + ExecClearTuple(slot); + publist = list_append_unique(publist, makeString(pubname)); + } + + if (publist) + { + StringInfo pubnames = makeStringInfo(); + + /* Prepare the list of publication(s) for warning message. */ + GetPublicationsStr(publist, pubnames, false); + ereport(ERROR, + errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("subscription \"%s\" with copy_data = true cannot subscribe to a publication with publish_via_partition_root = true and publishes partitioned table with foreign table as partition", + subname), + errdetail_plural("The subscription being created on a publication (%s) with publish_via_partition_root = true and contains partitioned tables with foreign table as partition ", + "The subscription being created on publications (%s) with publish_via_partition_root = true and contains partitioned tables with foreign table as partition ", + list_length(publist), pubnames->data), + errhint("Drop the foreign table from the publication or set publish_via_partition_root = false on publication or set copy_data = false.")); + } + + ExecDropSingleTupleTableSlot(slot); + + walrcv_clear_result(res); +} diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index ea96947d813..0f7f1bd4b42 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -1133,6 +1133,52 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, errmsg("\"%s\" is not partitioned", RelationGetRelationName(parent)))); + /* + * If we're creating a partition that's a foreign table, verify that + * the parent table is not in a publication with + * publish_via_partition_root enabled. + */ + if (rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE) + { + Oid schemaid; + List *puboids; + List *ancestors; + + /* Start with publications of all tables */ + puboids = GetAllTablesPublications(); + + /* capture all publications that include this relation directly */ + puboids = list_concat(puboids, GetRelationPublications(parent->rd_id)); + schemaid = RelationGetNamespace(parent); + puboids = list_concat(puboids, GetSchemaPublications(schemaid)); + + /* and do the same for its ancestors, if any */ + ancestors = get_partition_ancestors(parent->rd_id); + foreach_oid(ancestor, ancestors) + { + puboids = list_concat(puboids, GetRelationPublications(ancestor)); + schemaid = get_rel_namespace(ancestor); + puboids = list_concat(puboids, GetSchemaPublications(schemaid)); + } + + /* Check the publish_via_partition_root bit for each of those */ + list_sort(puboids, list_oid_cmp); + list_deduplicate_oid(puboids); + foreach_oid(puboid, puboids) + { + Publication *pub = GetPublication(puboid); + + if (pub->pubviaroot) + ereport(ERROR, + errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot create foreign table \"%s\" as a partition of \"%s\"", + RelationGetRelationName(rel), RelationGetRelationName(parent)), + errdetail("Partitioned table \"%s\" is published with option \"%s\" in publication \"%s\".", + RelationGetRelationName(parent), + "publish_via_partition_root", pub->name)); + } + } + /* * The partition constraint of the default partition depends on the * partition bounds of every other partition. It is possible that @@ -20317,6 +20363,70 @@ ATExecAttachPartition(List **wqueue, Relation rel, PartitionCmd *cmd, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot attach temporary relation of another session as partition"))); + /* + * If the relation to attach is a foreign table, or a partitioned table + * that contains a foreign table as partition, then verify that the parent + * table is not in a publication with publish_via_partition_root enabled. + */ + if (attachrel->rd_rel->relkind == RELKIND_FOREIGN_TABLE || + (attachrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE && + RelationHasForeignPartition(attachrel))) + { + Oid schemaid; + List *puboids; + List *ancestors; + + /* Start with publications of all tables */ + puboids = GetAllTablesPublications(); + + /* capture all publications that include this relation directly */ + puboids = list_concat(puboids, GetRelationPublications(rel->rd_id)); + schemaid = RelationGetNamespace(rel); + puboids = list_concat(puboids, GetSchemaPublications(schemaid)); + + /* and do the same for its ancestors, if any */ + ancestors = get_partition_ancestors(rel->rd_id); + foreach_oid(ancestor, ancestors) + { + puboids = list_concat(puboids, GetRelationPublications(ancestor)); + schemaid = get_rel_namespace(ancestor); + puboids = list_concat(puboids, GetSchemaPublications(schemaid)); + } + + /* Now check the publish_via_partition_root bit for each of those */ + list_sort(puboids, list_oid_cmp); + list_deduplicate_oid(puboids); + foreach_oid(puboid, puboids) + { + Publication *pub; + + pub = GetPublication(puboid); + if (pub->pubviaroot) + { + if (attachrel->rd_rel->relkind == RELKIND_FOREIGN_TABLE) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot attach foreign table \"%s\" to partition table \"%s\"", + RelationGetRelationName(attachrel), + RelationGetRelationName(rel)), + errdetail("Partitioned table \"%s\" is published with option \"%s\" in publication \"%s\".", + RelationGetRelationName(rel), + "publish_via_partition_root", + pub->name))); + else + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot attach table \"%s\" with a partition that's a foreign table to partition table \"%s\"", + RelationGetRelationName(attachrel), + RelationGetRelationName(rel)), + errdetail("Partitioned table \"%s\" is published with option \"%s\" in publication \"%s\".", + RelationGetRelationName(rel), + "publish_via_partition_root", + pub->name))); + } + } + } + /* * Check if attachrel has any identity columns or any columns that aren't * in the parent. diff --git a/src/backend/partitioning/partdesc.c b/src/backend/partitioning/partdesc.c index 328b4d450e4..56d176625e6 100644 --- a/src/backend/partitioning/partdesc.c +++ b/src/backend/partitioning/partdesc.c @@ -506,3 +506,35 @@ get_default_oid_from_partdesc(PartitionDesc partdesc) return InvalidOid; } + +/* + * Return true if the given partitioned table ultimately contains a + * partition that is a foreign table, false otherwise. + */ +bool +RelationHasForeignPartition(Relation rel) +{ + PartitionDesc pd = RelationGetPartitionDesc(rel, true); + + for (int i = 0; i < pd->nparts; i++) + { + if (pd->is_leaf[i]) + { + if (get_rel_relkind(pd->oids[i]) == RELKIND_FOREIGN_TABLE) + return true; + } + else + { + Relation part; + bool ret; + + part = table_open(pd->oids[i], NoLock); + ret = RelationHasForeignPartition(part); + table_close(part, NoLock); + if (ret) + return true; + } + } + + return false; +} diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index 940fc77fc2e..2cf407a3169 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -31,6 +31,7 @@ static void check_new_cluster_logical_replication_slots(void); static void check_new_cluster_subscription_configuration(void); static void check_old_cluster_for_valid_slots(void); static void check_old_cluster_subscription_state(void); +static void check_for_valid_publication(ClusterInfo *cluster); /* * DataTypesUsageChecks - definitions of data type checks for the old cluster @@ -633,6 +634,8 @@ check_and_dump_old_cluster(void) check_old_cluster_subscription_state(); } + check_for_valid_publication(&old_cluster); + check_for_data_types_usage(&old_cluster); /* @@ -2272,3 +2275,89 @@ check_old_cluster_subscription_state(void) else check_ok(); } + +/* + * Callback function for processing results of query for + * check_for_valid_publication()'s UpgradeTask. If the query returned + * any rows (i.e., the check failed), write the details to the report file. + */ +static void +process_pub_check(DbInfo *dbinfo, PGresult *res, void *arg) +{ + UpgradeTaskReport *report = (UpgradeTaskReport *) arg; + int ntups = PQntuples(res); + int i_puboid = PQfnumber(res, "oid"); + int i_pubviaroot = PQfnumber(res, "pubviaroot"); + int i_puballtables = PQfnumber(res, "puballtables"); + int i_ftrelid = PQfnumber(res, "ftrelid"); + + if (ntups == 0) + return; + + if (report->file == NULL && + (report->file = fopen_priv(report->path, "w")) == NULL) + pg_fatal("could not open file \"%s\": %m", report->path); + + fprintf(report->file, "In database: %s\n", dbinfo->db_name); + + for (int rowno = 0; rowno < ntups; rowno++) + fprintf(report->file, "The publication with oid: %s, has foreign table with oid: %s as partition of a published table. " + "Publication is published with publish_via_partition_root = %s and it is published %s.\n", + PQgetvalue(res, rowno, i_puboid), + PQgetvalue(res, rowno, i_ftrelid), + strcmp(PQgetvalue(res, rowno, i_pubviaroot), "f") ? "true" : "false", + strcmp(PQgetvalue(res, rowno, i_puballtables), "f") ? "FOR ALL TABLES" : "FOR TABLE/FOR TABLES IN SCHEMA"); +} + +/* + * check_for_valid_publication + * + * Verify if the publication has any published foreign table which are + * partitions of a table. + * + * For PG 18 and below foreign tables can be indirectly published when it is + * partition of a partitioned table. Avoid upgrading the cluster if it has a + * publication with publish_via_partition_root = true and has foreign table + * which is partition of a published table. + */ +static void +check_for_valid_publication(ClusterInfo *cluster) +{ + UpgradeTaskReport report; + UpgradeTask *task; + const char *query; + + if (GET_MAJOR_VERSION(cluster->major_version) >= 1900) + return; + + prep_status("Checking for invalid publications"); + + report.file = NULL; + snprintf(report.path, sizeof(report.path), "%s/%s", + log_opts.basedir, + "pubs_invalid.txt"); + + query = "SELECT p.oid, p.pubviaroot, p.puballtables, ft.ftrelid " + "from pg_catalog.pg_publication p, LATERAL " + "pg_get_publication_tables(p.pubname) gpt, LATERAL " + "pg_partition_tree(gpt.relid) gt JOIN pg_catalog.pg_foreign_table ft ON " + "ft.ftrelid = gt.relid WHERE p.pubviaroot = true;"; + + task = upgrade_task_create(); + upgrade_task_add_step(task, query, + process_pub_check, + true, &report); + upgrade_task_run(task, cluster); + upgrade_task_free(task); + + if (report.file) + { + fclose(report.file); + pg_log(PG_REPORT, "fatal"); + pg_fatal("Your installation contains publications, which has foreign table which are partitions of published table.\n" + "A list of potentially-affected publications is in the file:\n" + " %s", report.path); + } + else + check_ok(); +} diff --git a/src/include/partitioning/partdesc.h b/src/include/partitioning/partdesc.h index 34533f7004c..5fbafdc06f9 100644 --- a/src/include/partitioning/partdesc.h +++ b/src/include/partitioning/partdesc.h @@ -71,5 +71,6 @@ extern PartitionDesc PartitionDirectoryLookup(PartitionDirectory, Relation); extern void DestroyPartitionDirectory(PartitionDirectory pdir); extern Oid get_default_oid_from_partdesc(PartitionDesc partdesc); +extern bool RelationHasForeignPartition(Relation rel); #endif /* PARTDESC_H */ diff --git a/src/test/subscription/Makefile b/src/test/subscription/Makefile index 50b65d8f6ea..11041f79667 100644 --- a/src/test/subscription/Makefile +++ b/src/test/subscription/Makefile @@ -14,6 +14,7 @@ top_builddir = ../../.. include $(top_builddir)/src/Makefile.global EXTRA_INSTALL = contrib/hstore +EXTRA_INSTALL += contrib/postgres_fdw export with_icu diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl index 61b0cb4aa1a..41cfdc86580 100644 --- a/src/test/subscription/t/013_partition.pl +++ b/src/test/subscription/t/013_partition.pl @@ -8,6 +8,9 @@ use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; +my $stdout; +my $stderr; + # setup my $node_publisher = PostgreSQL::Test::Cluster->new('publisher'); @@ -891,4 +894,140 @@ $result = $node_subscriber2->safe_psql('postgres', "SELECT a, b, c FROM tab5_1 ORDER BY 1"); is($result, qq(4||1), 'updates of tab5 replicated correctly'); +# Test with partitioned table with foreign table as its partition. +$node_publisher->safe_psql('postgres', "CREATE DATABASE db1"); +$node_publisher->safe_psql( + 'db1', + q{CREATE EXTENSION IF NOT EXISTS postgres_fdw; + CREATE SERVER fdw FOREIGN DATA WRAPPER postgres_fdw; + CREATE USER MAPPING FOR CURRENT_USER SERVER fdw;}); + +$node_publisher->safe_psql( + 'db1', + q{CREATE SCHEMA sc1; + CREATE SCHEMA sc2; + CREATE TABLE sc1.tab6(id int) PARTITION BY RANGE(id); + CREATE TABLE sc1.tab6_1 PARTITION OF sc1.tab6 FOR VALUES FROM (0) TO (5); + CREATE TABLE sc1.tab6_2 PARTITION OF sc1.tab6 FOR VALUES FROM (5) TO (15) PARTITION BY RANGE(id); + CREATE FOREIGN TABLE sc1.tab6_2_1 PARTITION OF sc1.tab6_2 FOR VALUES FROM (10) TO (15) SERVER fdw;} +); +$node_publisher->safe_psql( + 'db1', + q{CREATE PUBLICATION pub_foreign_1 FOR TABLE sc1.tab6 with (publish_via_partition_root); + CREATE PUBLICATION pub_foreign_2 FOR TABLES IN SCHEMA sc1 with (publish_via_partition_root); + CREATE PUBLICATION pub_foreign_3 FOR ALL TABLES with (publish_via_partition_root); + CREATE PUBLICATION pub_foreign_4 FOR TABLES IN SCHEMA sc2 with (publish_via_partition_root); + }); + +$node_subscriber1->safe_psql('postgres', "CREATE DATABASE db1"); +$node_subscriber1->safe_psql( + 'db1', + q{CREATE SCHEMA sc1; + CREATE SCHEMA sc2; + CREATE TABLE sc1.tab6(id int) PARTITION BY RANGE(id); + CREATE TABLE sc1.tab6_1 PARTITION OF sc1.tab6 FOR VALUES FROM (0) TO (5); + CREATE TABLE sc1.tab6_2 PARTITION OF sc1.tab6 FOR VALUES FROM (5) TO (15) PARTITION BY RANGE(id); + CREATE TABLE sc1.tab6_2_1 PARTITION OF sc1.tab6_2 FOR VALUES FROM (10) TO (15);} +); + +$publisher_connstr = $node_publisher->connstr . ' dbname=db1'; + +# Create subscription on publication which has partitioned table with foreign +# table as its partition. +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "CREATE SUBSCRIPTION sub_foreign_1 CONNECTION '$publisher_connstr' PUBLICATION pub_foreign_1;" +); +like( + $stderr, + qr/ERROR: subscription "sub_foreign_1" with copy_data = true cannot subscribe to a publication with publish_via_partition_root = true and publishes partitioned table with foreign table as partition/, + "Create subscription on a publication FOR TABLE on partitioned table with foreign table as its partition" +); + +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "CREATE SUBSCRIPTION sub_foreign_1 CONNECTION '$publisher_connstr' PUBLICATION pub_foreign_2;" +); +like( + $stderr, + qr/ERROR: subscription "sub_foreign_1" with copy_data = true cannot subscribe to a publication with publish_via_partition_root = true and publishes partitioned table with foreign table as partition/, + "Create subscription on a publication FOR TABLES IN SCHEMA on partitioned table with foreign table as its partition" +); +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "CREATE SUBSCRIPTION sub_foreign_1 CONNECTION '$publisher_connstr' PUBLICATION pub_foreign_3;" +); +like( + $stderr, + qr/ERROR: subscription "sub_foreign_1" with copy_data = true cannot subscribe to a publication with publish_via_partition_root = true and publishes partitioned table with foreign table as partition/, + "Create subscription on a publication FOR ALL TABLES on partitioned table with foreign table as its partition" +); + +# If publication is created with publish_via_partition_root = false, we can +# create subscription on it. But replication of partitioned table with foreign +# is skipped +$node_publisher->safe_psql('db1', + "ALTER PUBLICATION pub_foreign_1 SET (publish_via_partition_root = false)" +); +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "CREATE SUBSCRIPTION sub_foreign_1 CONNECTION '$publisher_connstr' PUBLICATION pub_foreign_1;" +); +$node_subscriber1->safe_psql('db1', 'DROP SUBSCRIPTION sub_foreign_1'); + +# if subscription is created with copy_data=false, we can create subscription +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "CREATE SUBSCRIPTION sub_foreign_1 CONNECTION '$publisher_connstr' PUBLICATION pub_foreign_2 WITH (copy_data=false)" +); +$node_subscriber1->safe_psql('db1', 'DROP SUBSCRIPTION sub_foreign_1'); + +# Alter subscription to include publisher which has partitioned table with +# foreign table as its partition. +$node_subscriber1->safe_psql('db1', + "CREATE SUBSCRIPTION sub_foreign_1 CONNECTION '$publisher_connstr' PUBLICATION pub_foreign_4;" +); +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "ALTER SUBSCRIPTION sub_foreign_1 ADD PUBLICATION pub_foreign_2;"); +like( + $stderr, + qr/ERROR: subscription "sub_foreign_1" with copy_data = true cannot subscribe to a publication with publish_via_partition_root = true and publishes partitioned table with foreign table as partition/, + "Add publication which has partitioned table with foreign table as its partition to subscription" +); +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "ALTER SUBSCRIPTION sub_foreign_1 SET PUBLICATION pub_foreign_2;"); +like( + $stderr, + qr/ERROR: subscription "sub_foreign_1" with copy_data = true cannot subscribe to a publication with publish_via_partition_root = true and publishes partitioned table with foreign table as partition/, + "set publication which has partitioned table with foreign table as its partition for subscription" +); + +# Create foreign table as a partition of partitioned table already part of +# publication. +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "CREATE FOREIGN TABLE sc1.tab6_2_1 PARTITION OF sc1.tab6_2 FOR VALUES FROM (10) TO (15) SERVER fdw;" +); +$node_publisher->safe_psql('db1', "DROP FOREIGN TABLE sc1.tab6_2_1"); +($result, $stdout, $stderr) = $node_publisher->psql('db1', + "CREATE FOREIGN TABLE sc1.tab6_2_1 PARTITION OF sc1.tab6_2 FOR VALUES FROM (10) TO (15) SERVER fdw" +); +like( + $stderr, + qr/ERROR: cannot create foreign table "tab6_2_1" as a partition of "tab6_2"/, + "Create foreign table as a partition of partitioned table already part of publication" +); + +# Attach foreign table as a partition of partitioned table already part of +# publication. +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "CREATE FOREIGN TABLE sc1.tab6_2_1 (id int) SERVER fdw"); +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "ALTER TABLE sc1.tab6_2 ATTACH PARTITION sc1.tab6_2_1 FOR VALUES FROM (10) TO (15)" +); +$node_publisher->safe_psql('db1', + "CREATE FOREIGN TABLE sc1.tab6_2_1 (id int) SERVER fdw"); +($result, $stdout, $stderr) = $node_publisher->psql('db1', + "ALTER TABLE sc1.tab6_2 ATTACH PARTITION sc1.tab6_2_1 FOR VALUES FROM (10) TO (15)" +); +like( + $stderr, + qr/ERROR: cannot attach foreign table "tab6_2_1" to partition table "tab6_2"/, + "Attach foreign table as a partition of already published partitioned table" +); + done_testing(); -- 2.34.1 ^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: Restrict publishing of partitioned table with a foreign table as partition 2025-04-28 12:48 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-04-28 14:27 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> 2025-05-09 14:55 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-05-10 17:59 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> 2025-05-15 12:48 ` Re: Restrict publishing of partitioned table with a foreign table as partition Amit Kapila <[email protected]> 2025-05-19 16:32 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-06-04 10:41 ` Re: Restrict publishing of partitioned table with a foreign table as partition Ajin Cherian <[email protected]> 2025-06-09 05:57 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> @ 2025-07-01 02:50 ` Ajin Cherian <[email protected]> 2025-07-04 05:03 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 0 siblings, 1 reply; 19+ messages in thread From: Ajin Cherian @ 2025-07-01 02:50 UTC (permalink / raw) To: Shlok Kyal <[email protected]>; +Cc: Amit Kapila <[email protected]>; Álvaro Herrera <[email protected]>; vignesh C <[email protected]>; Sergey Tatarintsev <[email protected]>; [email protected] On Mon, Jun 9, 2025 at 3:58 PM Shlok Kyal <[email protected]> wrote: > > On Wed, 4 Jun 2025 at 16:12, Ajin Cherian <[email protected]> wrote: > > > > On Tue, May 20, 2025 at 2:33 AM Shlok Kyal <[email protected]> wrote: > > > > > > This approach seems better to me. I have created a patch with the > > > above approach. > > > > > > Thanks and Regards, > > > Shlok Kyal > > > > Some quick comments on the patch: > > 1. In doc/src/sgml/ref/create_subscription.sgml: > > + has partitioned table with foreign table as partition. If this scenario is > > + detected we ERROR is logged to the user. > > + </para> > > + > > > > Should be: "If this scenario is detected an ERROR is logged to the > > user." (remove "we"). > > > > In src/backend/commands/subscriptioncmds.c: > > 2. The comment header: > > + * This function is in charge of detecting if publisher with > > + * publish_via_partition_root=true publishes a partitioned table that has a > > + * foreign table as a partition. > > > > Add "and throw an error if found" at the end of that sentence to > > correctly describe what the function does. > > > > 3. > > + appendStringInfoString(&cmd, > > + "SELECT DISTINCT P.pubname AS pubname " > > + "from pg_catalog.pg_publication p, LATERAL " > > + "pg_get_publication_tables(p.pubname) gpt, LATERAL " > > + "pg_partition_tree(gpt.relid) gt JOIN > > pg_catalog.pg_foreign_table ft ON " > > + "ft.ftrelid = gt.relid WHERE p.pubviaroot > > = true AND p.pubname IN ("); > > > > use FROM rather than from to maintain SQL style consistency. > > > > 4. > > + errdetail_plural("The subscription being created on a > > publication (%s) with publish_via_root_partition = true and contains > > partitioned tables with foreign table as partition ", > > + "The subscription being created on > > publications (%s) with publish_via_root_partition = true and contains > > partitioned tables with foreign table as partition ", > > + list_length(publist), pubnames->data), > > > > I think you meant "publish_via_partition_root" here and not > > "publish_via_root_partition ". > > > > I have addressed all the comments and attached the updated patch. Hi Shlok, Some more comments: 1. + Replication is not supported for foreign tables. When used as partitions + of partitioned tables, publishing of the partitioned table is only allowed + if the <literal>publish_via_partition_root</literal> is set to + <literal>false</literal>. In patch: "When used as partitions of partitioned tables, publishing of the partitioned table is only allowed if the <literal>publish_via_partition_root</literal> is set to <literal>false</literal>. Change to: When foreign tables are used as partitions of partitioned tables, publishing of the partitioned table is only allowed if the <literal>publish_via_partition_root</literal> is set to <literal>false</literal>. 2. + <para> + When using a subscription parameter copy_data = true, corresponding + publications are checked if it has publish_via_partition_root = true and + has partitioned table with foreign table as partition. If this scenario is + detected an ERROR is logged to the user. + </para> In patch: "When using a subscription parameter copy_data = true, ..." Change to: "When using the subscription parameter copy_data = true, ..." In patch: "and has partitioned table with foreign table as partition." Change to: "and has a partitioned table with a foreign table as its partition" 3. + * check_publications_foreign_parts + * Check if the publications, on which subscriber is subscribing, publishes any + * partitioned table that has an foreign table as its partition and has + * publish_via_partition_root set as true. The check is performed only if + * copy_data is set as true for the subscription. In patch: "publishes any partitioned table that has an foreign table as its partition" Change to: "publishes any partitioned table that has a foreign table as its partition" 4. + * DML data changes are not published for data in foreign tables, + * and yet the tablesync worker is not smart enough to omit data from + * foreign tables when they are partitions of partitioned tables. change to:"Although DML changes to foreign tables are excluded from publication, the tablesync worker will still attempt to copy data from foreign table partitions during initial table synchronization." 5. + * When publish_via_partition_root is false, each partition published for + * replication is listed individually in pg_subscription_rel, and we + * don't add partitions that are foreign tables, so this check is not + * needed. In patch: "so this check is not needed" Change to: "so this function is not called for such tables" 6. + errdetail_plural("The subscription being created on a publication (%s) with publish_via_partition_root = true and contains partitioned tables with foreign table as partition ", + "The subscription being created on publications (%s) with publish_via_partition_root = true and contains partitioned tables with foreign table as partition ", + list_length(publist), pubnames->data), Change to: "The subscription is for a publication (%s) with publish_via_partition_root = true but one of the partitioned tables has a foreign table as a partition" "The subscription is for publications (%s) with publish_via_partition_root = true but one of the partitioned tables has a foreign table as a partition" 7. + /* capture all publications that include this relation directly */ + puboids = list_concat(puboids, GetRelationPublications(rel->rd_id)); + schemaid = RelationGetNamespace(rel); + puboids = list_concat(puboids, GetSchemaPublications(schemaid)); + + /* and do the same for its ancestors, if any */ + ancestors = get_partition_ancestors(rel->rd_id); + foreach_oid(ancestor, ancestors) + { + puboids = list_concat(puboids, GetRelationPublications(ancestor)); + schemaid = get_rel_namespace(ancestor); + puboids = list_concat(puboids, GetSchemaPublications(schemaid)); + } + + /* Now check the publish_via_partition_root bit for each of those */ + list_sort(puboids, list_oid_cmp); + list_deduplicate_oid(puboids); + foreach_oid(puboid, puboids) Why do we need to do all this logic for non partition foreign tables? Just directly throw an error for those tables and do this extra logic only for partitioned tables. 8. + pg_fatal("Your installation contains publications, which has foreign table which are partitions of published table.\n" + "A list of potentially-affected publications is in the file:\n" Change to: "Your installation contains publications, where one of the partitioned tables has a foreign table as a partition.\n" regards, Ajin Cherian Fujitsu Australia ^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: Restrict publishing of partitioned table with a foreign table as partition 2025-04-28 12:48 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-04-28 14:27 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> 2025-05-09 14:55 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-05-10 17:59 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> 2025-05-15 12:48 ` Re: Restrict publishing of partitioned table with a foreign table as partition Amit Kapila <[email protected]> 2025-05-19 16:32 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-06-04 10:41 ` Re: Restrict publishing of partitioned table with a foreign table as partition Ajin Cherian <[email protected]> 2025-06-09 05:57 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-07-01 02:50 ` Re: Restrict publishing of partitioned table with a foreign table as partition Ajin Cherian <[email protected]> @ 2025-07-04 05:03 ` Shlok Kyal <[email protected]> 0 siblings, 0 replies; 19+ messages in thread From: Shlok Kyal @ 2025-07-04 05:03 UTC (permalink / raw) To: Ajin Cherian <[email protected]>; +Cc: Amit Kapila <[email protected]>; Álvaro Herrera <[email protected]>; vignesh C <[email protected]>; Sergey Tatarintsev <[email protected]>; [email protected] On Tue, 1 Jul 2025 at 08:20, Ajin Cherian <[email protected]> wrote: > > On Mon, Jun 9, 2025 at 3:58 PM Shlok Kyal <[email protected]> wrote: > > > > On Wed, 4 Jun 2025 at 16:12, Ajin Cherian <[email protected]> wrote: > > > > > > On Tue, May 20, 2025 at 2:33 AM Shlok Kyal <[email protected]> wrote: > > > > > > > > This approach seems better to me. I have created a patch with the > > > > above approach. > > > > > > > > Thanks and Regards, > > > > Shlok Kyal > > > > > > Some quick comments on the patch: > > > 1. In doc/src/sgml/ref/create_subscription.sgml: > > > + has partitioned table with foreign table as partition. If this scenario is > > > + detected we ERROR is logged to the user. > > > + </para> > > > + > > > > > > Should be: "If this scenario is detected an ERROR is logged to the > > > user." (remove "we"). > > > > > > In src/backend/commands/subscriptioncmds.c: > > > 2. The comment header: > > > + * This function is in charge of detecting if publisher with > > > + * publish_via_partition_root=true publishes a partitioned table that has a > > > + * foreign table as a partition. > > > > > > Add "and throw an error if found" at the end of that sentence to > > > correctly describe what the function does. > > > > > > 3. > > > + appendStringInfoString(&cmd, > > > + "SELECT DISTINCT P.pubname AS pubname " > > > + "from pg_catalog.pg_publication p, LATERAL " > > > + "pg_get_publication_tables(p.pubname) gpt, LATERAL " > > > + "pg_partition_tree(gpt.relid) gt JOIN > > > pg_catalog.pg_foreign_table ft ON " > > > + "ft.ftrelid = gt.relid WHERE p.pubviaroot > > > = true AND p.pubname IN ("); > > > > > > use FROM rather than from to maintain SQL style consistency. > > > > > > 4. > > > + errdetail_plural("The subscription being created on a > > > publication (%s) with publish_via_root_partition = true and contains > > > partitioned tables with foreign table as partition ", > > > + "The subscription being created on > > > publications (%s) with publish_via_root_partition = true and contains > > > partitioned tables with foreign table as partition ", > > > + list_length(publist), pubnames->data), > > > > > > I think you meant "publish_via_partition_root" here and not > > > "publish_via_root_partition ". > > > > > > > I have addressed all the comments and attached the updated patch. > > > Hi Shlok, > > Some more comments: > 1. > + Replication is not supported for foreign tables. When used as partitions > + of partitioned tables, publishing of the partitioned table is only allowed > + if the <literal>publish_via_partition_root</literal> is set to > + <literal>false</literal>. > > In patch: "When used as partitions of partitioned tables, publishing > of the partitioned table is only allowed if the > <literal>publish_via_partition_root</literal> is set to > <literal>false</literal>. > > Change to: When foreign tables are used as partitions of partitioned > tables, publishing of the partitioned table is only allowed if the > <literal>publish_via_partition_root</literal> is set to > <literal>false</literal>. > Fixed > 2. > + <para> > + When using a subscription parameter copy_data = true, corresponding > + publications are checked if it has publish_via_partition_root = true and > + has partitioned table with foreign table as partition. If this scenario is > + detected an ERROR is logged to the user. > + </para> > > In patch: "When using a subscription parameter copy_data = true, ..." > Change to: "When using the subscription parameter copy_data = true, ..." > > In patch: "and has partitioned table with foreign table as partition." > Change to: "and has a partitioned table with a foreign table as its partition" > Fixed > 3. > + * check_publications_foreign_parts > + * Check if the publications, on which subscriber is subscribing, publishes any > + * partitioned table that has an foreign table as its partition and has > + * publish_via_partition_root set as true. The check is performed only if > + * copy_data is set as true for the subscription. > > In patch: "publishes any partitioned table that has an foreign table > as its partition" > Change to: "publishes any partitioned table that has a foreign table > as its partition" > Fixed > 4. > + * DML data changes are not published for data in foreign tables, > + * and yet the tablesync worker is not smart enough to omit data from > + * foreign tables when they are partitions of partitioned tables. > > change to:"Although DML changes to foreign tables are excluded from > publication, the tablesync worker will still attempt to copy data from > foreign table partitions during initial table synchronization." > Fixed > 5. > + * When publish_via_partition_root is false, each partition published for > + * replication is listed individually in pg_subscription_rel, and we > + * don't add partitions that are foreign tables, so this check is not > + * needed. > > In patch: "so this check is not needed" > Change to: "so this function is not called for such tables" > Fixed > 6. > + errdetail_plural("The subscription being created on a > publication (%s) with publish_via_partition_root = true and contains > partitioned tables with foreign table as partition ", > + "The subscription being created on > publications (%s) with publish_via_partition_root = true and contains > partitioned tables with foreign table as partition ", > + list_length(publist), pubnames->data), > > Change to: "The subscription is for a publication (%s) with > publish_via_partition_root = true but one of the partitioned tables > has a foreign table as a partition" > "The subscription is for publications (%s) with > publish_via_partition_root = true but one of the partitioned tables > has a foreign table as a partition" > "one of the partitioned tables" this would be misleading as publication can have multiple partitioned table which have foreign table as its partition. I have reworded it like: "The subscription is for a publication (%s) with publish_via_partition_root = true, but one or more partitioned tables have foreign tables as partitions.", "The subscription is for publications (%s) with publish_via_partition_root = true but one or more partitioned tables have foreign tables as partitions." > 7. > + /* capture all publications that include this relation directly */ > + puboids = list_concat(puboids, GetRelationPublications(rel->rd_id)); > + schemaid = RelationGetNamespace(rel); > + puboids = list_concat(puboids, GetSchemaPublications(schemaid)); > + > + /* and do the same for its ancestors, if any */ > + ancestors = get_partition_ancestors(rel->rd_id); > + foreach_oid(ancestor, ancestors) > + { > + puboids = list_concat(puboids, GetRelationPublications(ancestor)); > + schemaid = get_rel_namespace(ancestor); > + puboids = list_concat(puboids, GetSchemaPublications(schemaid)); > + } > + > + /* Now check the publish_via_partition_root bit for each of those */ > + list_sort(puboids, list_oid_cmp); > + list_deduplicate_oid(puboids); > + foreach_oid(puboid, puboids) > > Why do we need to do all this logic for non partition foreign tables? > Just directly throw an error for those tables and do this extra logic > only for partitioned tables. > We need to throw an error if foreign table is being attached to a partitioned table which is already published (or its parent is already published) with publish_via_partition_root is true. So, this logic is required to check if the table to which it is being attached is published with publish_via_partition_root or not. > > 8. > + pg_fatal("Your installation contains publications, which has > foreign table which are partitions of published table.\n" > + "A list of potentially-affected publications is in > the file:\n" > > Change to: "Your installation contains publications, where one of the > partitioned tables has a foreign table as a partition.\n" > There can be mutiple partitioned table published. Reworded the error as: "Your installation contains publications where one or more partitioned tables have foreign tables as partitions.\n" Thanks for reviewing the patch. I have addressed the comments and attached the latest patch. Thanks and Regards, Shlok Kyal Attachments: [application/octet-stream] v17-0001-Restrict-publishing-of-partitioned-table-with-fo.patch (29.2K, ../../CANhcyEW_s2LD6RiDSMHtWQnpYB67EWXqf7N8mn7dOrnaKMfROg@mail.gmail.com/2-v17-0001-Restrict-publishing-of-partitioned-table-with-fo.patch) download | inline diff: From 9339ce3b0ec511f30e21613de12076b35cfb196d Mon Sep 17 00:00:00 2001 From: Shlok Kyal <[email protected]> Date: Mon, 19 May 2025 19:47:04 +0530 Subject: [PATCH v17] Restrict publishing of partitioned table with foreign table as its partition Logical replication of foreign table is not supported and we throw an error in this case. But when we create a publication on a partitioned table that has a foreign table as its partition, the initial sync of such table is successful and we should avoid such cases. Current Behaviour in HEAD, when publication is created: 1. with publish_via_partition_root = true The root table is published, and initial data from partitions that are foreign tables is replicated. 2. with publish_via_partition_root = false and FOR ALL TABLES All leaf tables except partitions that are foreign tables are published. 3. with publish_via_partition_root = false and FOR TABLE/ FOR TABLES IN SCHEMA All leaf tables are published, including initial data from partitions that are foreign tables. With this patch we have following behaviour: 1. with publish_via_partition_root = true An error is thrown when we try to create subscription (with copy_data = true) on publication with partitioned table with foreign partition. When copy_data = false, we donot need this check as initial sync is not performed. Also we restrict creating/attaching a foreign table as partition of a partitioned table which is part of any publication. 2. with publish_via_partition_root = false We skip publishing partitions that are foreign tables. This is done by avoid adding such partitions in pg_subscription_rel catalog table. We have introduced two functions 'RelationHasForeignPartition' and 'check_publications_foreign_parts'. In 'RelationHasForeignPartition' we go through the child nodes of a partition and check if it has a foreign table. In 'check_publications_foreign_parts' we check if a subscription with copy_data = true is subscribing to a publication which has partitioned table with foreign table as its partition. --- doc/src/sgml/logical-replication.sgml | 10 +- doc/src/sgml/ref/create_subscription.sgml | 7 ++ src/backend/catalog/pg_publication.c | 26 ++-- src/backend/commands/subscriptioncmds.c | 100 ++++++++++++++++ src/backend/commands/tablecmds.c | 110 +++++++++++++++++ src/backend/partitioning/partdesc.c | 32 +++++ src/bin/pg_upgrade/check.c | 89 ++++++++++++++ src/include/partitioning/partdesc.h | 1 + src/test/subscription/Makefile | 1 + src/test/subscription/t/013_partition.pl | 139 ++++++++++++++++++++++ 10 files changed, 499 insertions(+), 16 deletions(-) diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index c32e6bc000d..275dd296f18 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -2079,10 +2079,18 @@ CONTEXT: processing remote data for replication origin "pg_16395" during "INSER <listitem> <para> - Replication is only supported by tables, including partitioned tables. + Replication is only supported for tables, including partitioned tables. Attempts to replicate other types of relations, such as views, materialized views, or foreign tables, will result in an error. </para> + <para> + Replication is not supported for foreign tables. When foreign tables are + used as partitions of partitioned tables, publishing of the partitioned + table is only allowed if the <literal>publish_via_partition_root</literal> + is set to <literal>false</literal>. In this mode, changes to a partition + that is a foreign table are ignored for the purposes of replication, and + data contained in them is not included during initial synchronization. + </para> </listitem> <listitem> diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml index 57dec28a5df..9019fe1e384 100644 --- a/doc/src/sgml/ref/create_subscription.sgml +++ b/doc/src/sgml/ref/create_subscription.sgml @@ -545,6 +545,13 @@ WHERE C.relnamespace = N.oid AND PT.pubname IN (<pub-names>); </programlisting></para> + <para> + When using the subscription parameter copy_data = true, corresponding + publications are checked if it has publish_via_partition_root = true and + has a partitioned table with a foreign table as its partition. If this + scenario is detected an ERROR is logged to the user. + </para> + </refsect1> <refsect1> diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c index d6f94db5d99..56255dcd5cc 100644 --- a/src/backend/catalog/pg_publication.c +++ b/src/backend/catalog/pg_publication.c @@ -304,7 +304,7 @@ check_and_fetch_column_list(Publication *pub, Oid relid, MemoryContext mcxt, /* * Gets the relations based on the publication partition option for a specified - * relation. + * relation. Foreign tables are not included. */ List * GetPubPartitionOptionRelations(List *result, PublicationPartOpt pub_partopt, @@ -313,25 +313,21 @@ GetPubPartitionOptionRelations(List *result, PublicationPartOpt pub_partopt, if (get_rel_relkind(relid) == RELKIND_PARTITIONED_TABLE && pub_partopt != PUBLICATION_PART_ROOT) { - List *all_parts = find_all_inheritors(relid, NoLock, - NULL); + List *all_parts = find_all_inheritors(relid, NoLock, NULL); - if (pub_partopt == PUBLICATION_PART_ALL) - result = list_concat(result, all_parts); - else if (pub_partopt == PUBLICATION_PART_LEAF) + foreach_oid(partOid, all_parts) { - ListCell *lc; + char relkind = get_rel_relkind(partOid); - foreach(lc, all_parts) - { - Oid partOid = lfirst_oid(lc); + if (relkind == RELKIND_FOREIGN_TABLE) + continue; - if (get_rel_relkind(partOid) != RELKIND_PARTITIONED_TABLE) - result = lappend_oid(result, partOid); - } + if (pub_partopt == PUBLICATION_PART_LEAF && + relkind == RELKIND_PARTITIONED_TABLE) + continue; + + result = lappend_oid(result, partOid); } - else - Assert(false); } else result = lappend_oid(result, relid); diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c index 4ff246cd943..83978b77d0c 100644 --- a/src/backend/commands/subscriptioncmds.c +++ b/src/backend/commands/subscriptioncmds.c @@ -112,6 +112,9 @@ static List *merge_publications(List *oldpublist, List *newpublist, bool addpub, static void ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err); static void CheckAlterSubOption(Subscription *sub, const char *option, bool slot_needs_update, bool isTopLevel); +static void check_publications_foreign_parts(WalReceiverConn *wrconn, + List *publications, bool copydata, + char *subname); /* @@ -723,6 +726,8 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, check_publications(wrconn, publications); check_publications_origin(wrconn, publications, opts.copy_data, opts.origin, NULL, 0, stmt->subname); + check_publications_foreign_parts(wrconn, publications, + opts.copy_data, stmt->subname); /* * Set sync state based on if we were asked to do data copy or @@ -883,6 +888,8 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data, check_publications_origin(wrconn, sub->publications, copy_data, sub->origin, subrel_local_oids, subrel_count, sub->name); + check_publications_foreign_parts(wrconn, sub->publications, copy_data, + sub->name); /* * Rels that we want to remove from subscription and drop any slots @@ -2507,3 +2514,96 @@ defGetStreamingMode(DefElem *def) def->defname))); return LOGICALREP_STREAM_OFF; /* keep compiler quiet */ } + +/* + * check_publications_foreign_parts + * Check if the publications, on which subscriber is subscribing, publishes any + * partitioned table that has a foreign table as its partition and has + * publish_via_partition_root set as true. The check is performed only if + * copy_data is set as true for the subscription. + * + * Although DML changes to foreign tables are excluded from publication, the + * tablesync worker will still attempt to copy data from foreign table + * partitions during initial table synchronization. To avoid the + * inconsistencies that would result, we disallow foreign tables from being + * published generally. However, it's possible for partitioned tables to have + * foreign tables as partitions, and we would like to allow publishing those + * partitioned tables so that the other partitions are replicated. + * + * This function is in charge of detecting if publisher with + * publish_via_partition_root=true publishes a partitioned table that has a + * foreign table as a partition and throw an error if found. + * + * When publish_via_partition_root is false, each partition published for + * replication is listed individually in pg_subscription_rel, and we + * don't add partitions that are foreign tables, so this function is not called + * for such tables. + */ +static void +check_publications_foreign_parts(WalReceiverConn *wrconn, List *publications, + bool copydata, char *subname) +{ + WalRcvExecResult *res; + StringInfoData cmd; + TupleTableSlot *slot; + Oid tableRow[1] = {TEXTOID}; + List *publist = NIL; + int i; + + if (!copydata) + return; + + initStringInfo(&cmd); + appendStringInfoString(&cmd, + "SELECT DISTINCT P.pubname AS pubname " + "FROM pg_catalog.pg_publication p, LATERAL " + "pg_get_publication_tables(p.pubname) gpt, LATERAL " + "pg_partition_tree(gpt.relid) gt JOIN pg_catalog.pg_foreign_table ft ON " + "ft.ftrelid = gt.relid WHERE p.pubviaroot = true AND p.pubname IN ("); + + GetPublicationsStr(publications, &cmd, true); + appendStringInfoString(&cmd, ")\n"); + + res = walrcv_exec(wrconn, cmd.data, 1, tableRow); + pfree(cmd.data); + + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + (errcode(ERRCODE_CONNECTION_FAILURE), + errmsg("could not receive list of replicated tables from the publisher: %s", + res->err))); + + /* Process tables. */ + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); + while (tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + { + char *pubname; + bool isnull; + + pubname = TextDatumGetCString(slot_getattr(slot, 1, &isnull)); + Assert(!isnull); + + ExecClearTuple(slot); + publist = list_append_unique(publist, makeString(pubname)); + } + + if (publist) + { + StringInfo pubnames = makeStringInfo(); + + /* Prepare the list of publication(s) for warning message. */ + GetPublicationsStr(publist, pubnames, false); + ereport(ERROR, + errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("subscription \"%s\" with copy_data = true cannot subscribe to a publication with publish_via_partition_root = true and publishes partitioned table with foreign table as partition", + subname), + errdetail_plural("The subscription is for a publication (%s) with publish_via_partition_root = true, but one or more partitioned tables have foreign tables as partitions.", + "The subscription is for publications (%s) with publish_via_partition_root = true but one or more partitioned tables have foreign tables as partitions.", + list_length(publist), pubnames->data), + errhint("Drop the foreign table from the publication or set publish_via_partition_root = false on publication or set copy_data = false.")); + } + + ExecDropSingleTupleTableSlot(slot); + + walrcv_clear_result(res); +} diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index b8837f26cb4..5859224900f 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -1133,6 +1133,52 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, errmsg("\"%s\" is not partitioned", RelationGetRelationName(parent)))); + /* + * If we're creating a partition that's a foreign table, verify that + * the parent table is not in a publication with + * publish_via_partition_root enabled. + */ + if (rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE) + { + Oid schemaid; + List *puboids; + List *ancestors; + + /* Start with publications of all tables */ + puboids = GetAllTablesPublications(); + + /* capture all publications that include this relation directly */ + puboids = list_concat(puboids, GetRelationPublications(parent->rd_id)); + schemaid = RelationGetNamespace(parent); + puboids = list_concat(puboids, GetSchemaPublications(schemaid)); + + /* and do the same for its ancestors, if any */ + ancestors = get_partition_ancestors(parent->rd_id); + foreach_oid(ancestor, ancestors) + { + puboids = list_concat(puboids, GetRelationPublications(ancestor)); + schemaid = get_rel_namespace(ancestor); + puboids = list_concat(puboids, GetSchemaPublications(schemaid)); + } + + /* Check the publish_via_partition_root bit for each of those */ + list_sort(puboids, list_oid_cmp); + list_deduplicate_oid(puboids); + foreach_oid(puboid, puboids) + { + Publication *pub = GetPublication(puboid); + + if (pub->pubviaroot) + ereport(ERROR, + errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot create foreign table \"%s\" as a partition of \"%s\"", + RelationGetRelationName(rel), RelationGetRelationName(parent)), + errdetail("Partitioned table \"%s\" is published with option \"%s\" in publication \"%s\".", + RelationGetRelationName(parent), + "publish_via_partition_root", pub->name)); + } + } + /* * The partition constraint of the default partition depends on the * partition bounds of every other partition. It is possible that @@ -20322,6 +20368,70 @@ ATExecAttachPartition(List **wqueue, Relation rel, PartitionCmd *cmd, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot attach temporary relation of another session as partition"))); + /* + * If the relation to attach is a foreign table, or a partitioned table + * that contains a foreign table as partition, then verify that the parent + * table is not in a publication with publish_via_partition_root enabled. + */ + if (attachrel->rd_rel->relkind == RELKIND_FOREIGN_TABLE || + (attachrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE && + RelationHasForeignPartition(attachrel))) + { + Oid schemaid; + List *puboids; + List *ancestors; + + /* Start with publications of all tables */ + puboids = GetAllTablesPublications(); + + /* capture all publications that include this relation directly */ + puboids = list_concat(puboids, GetRelationPublications(rel->rd_id)); + schemaid = RelationGetNamespace(rel); + puboids = list_concat(puboids, GetSchemaPublications(schemaid)); + + /* and do the same for its ancestors, if any */ + ancestors = get_partition_ancestors(rel->rd_id); + foreach_oid(ancestor, ancestors) + { + puboids = list_concat(puboids, GetRelationPublications(ancestor)); + schemaid = get_rel_namespace(ancestor); + puboids = list_concat(puboids, GetSchemaPublications(schemaid)); + } + + /* Now check the publish_via_partition_root bit for each of those */ + list_sort(puboids, list_oid_cmp); + list_deduplicate_oid(puboids); + foreach_oid(puboid, puboids) + { + Publication *pub; + + pub = GetPublication(puboid); + if (pub->pubviaroot) + { + if (attachrel->rd_rel->relkind == RELKIND_FOREIGN_TABLE) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot attach foreign table \"%s\" to partition table \"%s\"", + RelationGetRelationName(attachrel), + RelationGetRelationName(rel)), + errdetail("Partitioned table \"%s\" is published with option \"%s\" in publication \"%s\".", + RelationGetRelationName(rel), + "publish_via_partition_root", + pub->name))); + else + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot attach table \"%s\" with a partition that's a foreign table to partition table \"%s\"", + RelationGetRelationName(attachrel), + RelationGetRelationName(rel)), + errdetail("Partitioned table \"%s\" is published with option \"%s\" in publication \"%s\".", + RelationGetRelationName(rel), + "publish_via_partition_root", + pub->name))); + } + } + } + /* * Check if attachrel has any identity columns or any columns that aren't * in the parent. diff --git a/src/backend/partitioning/partdesc.c b/src/backend/partitioning/partdesc.c index 328b4d450e4..56d176625e6 100644 --- a/src/backend/partitioning/partdesc.c +++ b/src/backend/partitioning/partdesc.c @@ -506,3 +506,35 @@ get_default_oid_from_partdesc(PartitionDesc partdesc) return InvalidOid; } + +/* + * Return true if the given partitioned table ultimately contains a + * partition that is a foreign table, false otherwise. + */ +bool +RelationHasForeignPartition(Relation rel) +{ + PartitionDesc pd = RelationGetPartitionDesc(rel, true); + + for (int i = 0; i < pd->nparts; i++) + { + if (pd->is_leaf[i]) + { + if (get_rel_relkind(pd->oids[i]) == RELKIND_FOREIGN_TABLE) + return true; + } + else + { + Relation part; + bool ret; + + part = table_open(pd->oids[i], NoLock); + ret = RelationHasForeignPartition(part); + table_close(part, NoLock); + if (ret) + return true; + } + } + + return false; +} diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index fb063a2de42..d2e58c4463a 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -31,6 +31,7 @@ static void check_new_cluster_logical_replication_slots(void); static void check_new_cluster_subscription_configuration(void); static void check_old_cluster_for_valid_slots(void); static void check_old_cluster_subscription_state(void); +static void check_for_valid_publication(ClusterInfo *cluster); /* * DataTypesUsageChecks - definitions of data type checks for the old cluster @@ -634,6 +635,8 @@ check_and_dump_old_cluster(void) check_old_cluster_subscription_state(); } + check_for_valid_publication(&old_cluster); + check_for_data_types_usage(&old_cluster); /* @@ -2273,3 +2276,89 @@ check_old_cluster_subscription_state(void) else check_ok(); } + +/* + * Callback function for processing results of query for + * check_for_valid_publication()'s UpgradeTask. If the query returned + * any rows (i.e., the check failed), write the details to the report file. + */ +static void +process_pub_check(DbInfo *dbinfo, PGresult *res, void *arg) +{ + UpgradeTaskReport *report = (UpgradeTaskReport *) arg; + int ntups = PQntuples(res); + int i_puboid = PQfnumber(res, "oid"); + int i_pubviaroot = PQfnumber(res, "pubviaroot"); + int i_puballtables = PQfnumber(res, "puballtables"); + int i_ftrelid = PQfnumber(res, "ftrelid"); + + if (ntups == 0) + return; + + if (report->file == NULL && + (report->file = fopen_priv(report->path, "w")) == NULL) + pg_fatal("could not open file \"%s\": %m", report->path); + + fprintf(report->file, "In database: %s\n", dbinfo->db_name); + + for (int rowno = 0; rowno < ntups; rowno++) + fprintf(report->file, "The publication with oid: %s, has foreign table with oid: %s as partition of a published table. " + "Publication is published with publish_via_partition_root = %s and it is published %s.\n", + PQgetvalue(res, rowno, i_puboid), + PQgetvalue(res, rowno, i_ftrelid), + strcmp(PQgetvalue(res, rowno, i_pubviaroot), "f") ? "true" : "false", + strcmp(PQgetvalue(res, rowno, i_puballtables), "f") ? "FOR ALL TABLES" : "FOR TABLE/FOR TABLES IN SCHEMA"); +} + +/* + * check_for_valid_publication + * + * Verify if the publication has any published foreign table which are + * partitions of a table. + * + * For PG 18 and below foreign tables can be indirectly published when it is + * partition of a partitioned table. Avoid upgrading the cluster if it has a + * publication with publish_via_partition_root = true and has foreign table + * which is partition of a published table. + */ +static void +check_for_valid_publication(ClusterInfo *cluster) +{ + UpgradeTaskReport report; + UpgradeTask *task; + const char *query; + + if (GET_MAJOR_VERSION(cluster->major_version) >= 1900) + return; + + prep_status("Checking for invalid publications"); + + report.file = NULL; + snprintf(report.path, sizeof(report.path), "%s/%s", + log_opts.basedir, + "pubs_invalid.txt"); + + query = "SELECT p.oid, p.pubviaroot, p.puballtables, ft.ftrelid " + "from pg_catalog.pg_publication p, LATERAL " + "pg_get_publication_tables(p.pubname) gpt, LATERAL " + "pg_partition_tree(gpt.relid) gt JOIN pg_catalog.pg_foreign_table ft ON " + "ft.ftrelid = gt.relid WHERE p.pubviaroot = true;"; + + task = upgrade_task_create(); + upgrade_task_add_step(task, query, + process_pub_check, + true, &report); + upgrade_task_run(task, cluster); + upgrade_task_free(task); + + if (report.file) + { + fclose(report.file); + pg_log(PG_REPORT, "fatal"); + pg_fatal("Your installation contains publications where one or more partitioned tables have foreign tables as partitions.\n" + "A list of potentially-affected publications is in the file:\n" + " %s", report.path); + } + else + check_ok(); +} diff --git a/src/include/partitioning/partdesc.h b/src/include/partitioning/partdesc.h index 34533f7004c..5fbafdc06f9 100644 --- a/src/include/partitioning/partdesc.h +++ b/src/include/partitioning/partdesc.h @@ -71,5 +71,6 @@ extern PartitionDesc PartitionDirectoryLookup(PartitionDirectory, Relation); extern void DestroyPartitionDirectory(PartitionDirectory pdir); extern Oid get_default_oid_from_partdesc(PartitionDesc partdesc); +extern bool RelationHasForeignPartition(Relation rel); #endif /* PARTDESC_H */ diff --git a/src/test/subscription/Makefile b/src/test/subscription/Makefile index 50b65d8f6ea..11041f79667 100644 --- a/src/test/subscription/Makefile +++ b/src/test/subscription/Makefile @@ -14,6 +14,7 @@ top_builddir = ../../.. include $(top_builddir)/src/Makefile.global EXTRA_INSTALL = contrib/hstore +EXTRA_INSTALL += contrib/postgres_fdw export with_icu diff --git a/src/test/subscription/t/013_partition.pl b/src/test/subscription/t/013_partition.pl index 4f78dd48815..466473bd387 100644 --- a/src/test/subscription/t/013_partition.pl +++ b/src/test/subscription/t/013_partition.pl @@ -8,6 +8,9 @@ use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; +my $stdout; +my $stderr; + # setup my $node_publisher = PostgreSQL::Test::Cluster->new('publisher'); @@ -890,4 +893,140 @@ $result = $node_subscriber2->safe_psql('postgres', "SELECT a, b, c FROM tab5_1 ORDER BY 1"); is($result, qq(4||1), 'updates of tab5 replicated correctly'); +# Test with partitioned table with foreign table as its partition. +$node_publisher->safe_psql('postgres', "CREATE DATABASE db1"); +$node_publisher->safe_psql( + 'db1', + q{CREATE EXTENSION IF NOT EXISTS postgres_fdw; + CREATE SERVER fdw FOREIGN DATA WRAPPER postgres_fdw; + CREATE USER MAPPING FOR CURRENT_USER SERVER fdw;}); + +$node_publisher->safe_psql( + 'db1', + q{CREATE SCHEMA sc1; + CREATE SCHEMA sc2; + CREATE TABLE sc1.tab6(id int) PARTITION BY RANGE(id); + CREATE TABLE sc1.tab6_1 PARTITION OF sc1.tab6 FOR VALUES FROM (0) TO (5); + CREATE TABLE sc1.tab6_2 PARTITION OF sc1.tab6 FOR VALUES FROM (5) TO (15) PARTITION BY RANGE(id); + CREATE FOREIGN TABLE sc1.tab6_2_1 PARTITION OF sc1.tab6_2 FOR VALUES FROM (10) TO (15) SERVER fdw;} +); +$node_publisher->safe_psql( + 'db1', + q{CREATE PUBLICATION pub_foreign_1 FOR TABLE sc1.tab6 with (publish_via_partition_root); + CREATE PUBLICATION pub_foreign_2 FOR TABLES IN SCHEMA sc1 with (publish_via_partition_root); + CREATE PUBLICATION pub_foreign_3 FOR ALL TABLES with (publish_via_partition_root); + CREATE PUBLICATION pub_foreign_4 FOR TABLES IN SCHEMA sc2 with (publish_via_partition_root); + }); + +$node_subscriber1->safe_psql('postgres', "CREATE DATABASE db1"); +$node_subscriber1->safe_psql( + 'db1', + q{CREATE SCHEMA sc1; + CREATE SCHEMA sc2; + CREATE TABLE sc1.tab6(id int) PARTITION BY RANGE(id); + CREATE TABLE sc1.tab6_1 PARTITION OF sc1.tab6 FOR VALUES FROM (0) TO (5); + CREATE TABLE sc1.tab6_2 PARTITION OF sc1.tab6 FOR VALUES FROM (5) TO (15) PARTITION BY RANGE(id); + CREATE TABLE sc1.tab6_2_1 PARTITION OF sc1.tab6_2 FOR VALUES FROM (10) TO (15);} +); + +$publisher_connstr = $node_publisher->connstr . ' dbname=db1'; + +# Create subscription on publication which has partitioned table with foreign +# table as its partition. +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "CREATE SUBSCRIPTION sub_foreign_1 CONNECTION '$publisher_connstr' PUBLICATION pub_foreign_1;" +); +like( + $stderr, + qr/ERROR: subscription "sub_foreign_1" with copy_data = true cannot subscribe to a publication with publish_via_partition_root = true and publishes partitioned table with foreign table as partition/, + "Create subscription on a publication FOR TABLE on partitioned table with foreign table as its partition" +); + +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "CREATE SUBSCRIPTION sub_foreign_1 CONNECTION '$publisher_connstr' PUBLICATION pub_foreign_2;" +); +like( + $stderr, + qr/ERROR: subscription "sub_foreign_1" with copy_data = true cannot subscribe to a publication with publish_via_partition_root = true and publishes partitioned table with foreign table as partition/, + "Create subscription on a publication FOR TABLES IN SCHEMA on partitioned table with foreign table as its partition" +); +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "CREATE SUBSCRIPTION sub_foreign_1 CONNECTION '$publisher_connstr' PUBLICATION pub_foreign_3;" +); +like( + $stderr, + qr/ERROR: subscription "sub_foreign_1" with copy_data = true cannot subscribe to a publication with publish_via_partition_root = true and publishes partitioned table with foreign table as partition/, + "Create subscription on a publication FOR ALL TABLES on partitioned table with foreign table as its partition" +); + +# If publication is created with publish_via_partition_root = false, we can +# create subscription on it. But replication of partitioned table with foreign +# is skipped +$node_publisher->safe_psql('db1', + "ALTER PUBLICATION pub_foreign_1 SET (publish_via_partition_root = false)" +); +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "CREATE SUBSCRIPTION sub_foreign_1 CONNECTION '$publisher_connstr' PUBLICATION pub_foreign_1;" +); +$node_subscriber1->safe_psql('db1', 'DROP SUBSCRIPTION sub_foreign_1'); + +# if subscription is created with copy_data=false, we can create subscription +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "CREATE SUBSCRIPTION sub_foreign_1 CONNECTION '$publisher_connstr' PUBLICATION pub_foreign_2 WITH (copy_data=false)" +); +$node_subscriber1->safe_psql('db1', 'DROP SUBSCRIPTION sub_foreign_1'); + +# Alter subscription to include publisher which has partitioned table with +# foreign table as its partition. +$node_subscriber1->safe_psql('db1', + "CREATE SUBSCRIPTION sub_foreign_1 CONNECTION '$publisher_connstr' PUBLICATION pub_foreign_4;" +); +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "ALTER SUBSCRIPTION sub_foreign_1 ADD PUBLICATION pub_foreign_2;"); +like( + $stderr, + qr/ERROR: subscription "sub_foreign_1" with copy_data = true cannot subscribe to a publication with publish_via_partition_root = true and publishes partitioned table with foreign table as partition/, + "Add publication which has partitioned table with foreign table as its partition to subscription" +); +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "ALTER SUBSCRIPTION sub_foreign_1 SET PUBLICATION pub_foreign_2;"); +like( + $stderr, + qr/ERROR: subscription "sub_foreign_1" with copy_data = true cannot subscribe to a publication with publish_via_partition_root = true and publishes partitioned table with foreign table as partition/, + "set publication which has partitioned table with foreign table as its partition for subscription" +); + +# Create foreign table as a partition of partitioned table already part of +# publication. +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "CREATE FOREIGN TABLE sc1.tab6_2_1 PARTITION OF sc1.tab6_2 FOR VALUES FROM (10) TO (15) SERVER fdw;" +); +$node_publisher->safe_psql('db1', "DROP FOREIGN TABLE sc1.tab6_2_1"); +($result, $stdout, $stderr) = $node_publisher->psql('db1', + "CREATE FOREIGN TABLE sc1.tab6_2_1 PARTITION OF sc1.tab6_2 FOR VALUES FROM (10) TO (15) SERVER fdw" +); +like( + $stderr, + qr/ERROR: cannot create foreign table "tab6_2_1" as a partition of "tab6_2"/, + "Create foreign table as a partition of partitioned table already part of publication" +); + +# Attach foreign table as a partition of partitioned table already part of +# publication. +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "CREATE FOREIGN TABLE sc1.tab6_2_1 (id int) SERVER fdw"); +($result, $stdout, $stderr) = $node_subscriber1->psql('db1', + "ALTER TABLE sc1.tab6_2 ATTACH PARTITION sc1.tab6_2_1 FOR VALUES FROM (10) TO (15)" +); +$node_publisher->safe_psql('db1', + "CREATE FOREIGN TABLE sc1.tab6_2_1 (id int) SERVER fdw"); +($result, $stdout, $stderr) = $node_publisher->psql('db1', + "ALTER TABLE sc1.tab6_2 ATTACH PARTITION sc1.tab6_2_1 FOR VALUES FROM (10) TO (15)" +); +like( + $stderr, + qr/ERROR: cannot attach foreign table "tab6_2_1" to partition table "tab6_2"/, + "Attach foreign table as a partition of already published partitioned table" +); + done_testing(); -- 2.34.1 ^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: Restrict publishing of partitioned table with a foreign table as partition @ 2025-04-28 12:51 ` Shlok Kyal <[email protected]> 1 sibling, 0 replies; 19+ messages in thread From: Shlok Kyal @ 2025-04-28 12:51 UTC (permalink / raw) To: Sergey Tatarintsev <[email protected]>; +Cc: Álvaro Herrera <[email protected]>; vignesh C <[email protected]>; [email protected] On Mon, 7 Apr 2025 at 09:43, Sergey Tatarintsev <[email protected]> wrote: > > 07.04.2025 03:27, Álvaro Herrera пишет: > > On 2025-Apr-01, Shlok Kyal wrote: > > I have modified the comment in create_publication.sgml and also added > comment in the restrictions section of logical-replication.sgml. > I have also added a more detailed explanation in comment of > 'check_foreign_tables' > > I have attached the updated v11 patch. > > Sadly I don't have time to describe the changes proposed here right now, > but I'll do that early tomorrow. (Some minor changes are still needed, > particularly the comments to publication_check_foreign_parts which are > mostly unchanged from what your patch has. I'll do another review round > tomorrow.) > > Hello! > > I looked at the latest patch again and found one more place for list_free(). Also look at additional test case: > > diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c index 51e463c112b..7fcc191feb9 100644 --- a/src/backend/catalog/pg_publication.c +++ b/src/backend/catalog/pg_publication.c @@ -442,6 +442,7 @@ GetPubPartitionOptionRelations(List *result, PublicationPartOpt pub_partopt, result = lappend_oid(result, partOid); } + list_free(all_parts); } else result = lappend_oid(result, relid); diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql index 49c9d98b668..e56aebc397a 100644 --- a/src/test/regress/sql/publication.sql +++ b/src/test/regress/sql/publication.sql @@ -1296,6 +1296,14 @@ SELECT pubname, tablename FROM pg_publication_tables WHERE schemaname in ('sch3' -- foreign partition ALTER PUBLICATION pub1 SET (publish_via_partition_root); +CREATE SCHEMA sch5; +CREATE SCHEMA sch6; +CREATE TABLE sch6.tmain(id int) PARTITION BY RANGE(id); +CREATE TABLE sch5.part1 PARTITION OF sch6.tmain FOR VALUES FROM (0) TO (10) PARTITION BY RANGE(id); +CREATE FOREIGN TABLE sch6.part2 PARTITION OF sch5.part1 FOR VALUES FROM (0) TO (5) SERVER fdw_server; +CREATE PUBLICATION pub4 FOR TABLES IN SCHEMA sch5 WITH (publish_via_partition_root); +SELECT pubname, tablename FROM pg_publication_tables WHERE pubname = 'pub4' ORDER BY pubname, tablename; + DROP PUBLICATION pub1; DROP PUBLICATION pub2; DROP PUBLICATION pub3; > > I think this is a wrong assumption: > > ScanKeyInit(&key[keycount++], Anum_pg_class_relispartition, BTEqualStrategyNumber, F_BOOLEQ, BoolGetDatum(false)); > > In this case sch5.part1 is partitioned table, but it also partition of table in different schema > Hi Sergey, Thanks for reviewing the patch. I have addressed the comment in the latest patch shared in [1]. [1]: https://www.postgresql.org/message-id/CANhcyEW0QMiJXMqpPFRHni-q0Rm4R0hpZ0LdaqA%3DF3wvDUU6sQ%40mail.g... Thanks and Regards, Shlok Kyal ^ permalink raw reply [nested|flat] 19+ messages in thread
end of thread, other threads:[~2025-07-04 05:03 UTC | newest] Thread overview: 19+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2021-03-14 22:12 [PATCH 09/10] Add zstd compression levels Justin Pryzby <[email protected]> 2025-03-24 09:33 Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-03-24 15:47 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> 2025-03-28 09:43 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-03-28 11:04 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> 2025-04-01 14:48 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-04-04 05:06 ` Re: Restrict publishing of partitioned table with a foreign table as partition Sergey Tatarintsev <[email protected]> 2025-04-04 09:14 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-04-28 12:48 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-04-28 14:27 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> 2025-05-09 14:55 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-05-10 17:59 ` Re: Restrict publishing of partitioned table with a foreign table as partition Álvaro Herrera <[email protected]> 2025-05-15 12:48 ` Re: Restrict publishing of partitioned table with a foreign table as partition Amit Kapila <[email protected]> 2025-05-19 16:32 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-06-04 10:41 ` Re: Restrict publishing of partitioned table with a foreign table as partition Ajin Cherian <[email protected]> 2025-06-09 05:57 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-07-01 02:50 ` Re: Restrict publishing of partitioned table with a foreign table as partition Ajin Cherian <[email protected]> 2025-07-04 05:03 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]> 2025-04-28 12:51 ` Re: Restrict publishing of partitioned table with a foreign table as partition Shlok Kyal <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox