public inbox for [email protected]
help / color / mirror / Atom feed[PATCH v3 1/2] In-place table persistence change
58+ messages / 12 participants
[nested] [flat]
* [PATCH v3 1/2] In-place table persistence change
@ 2020-11-11 12:51 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 58+ messages in thread
From: Kyotaro Horiguchi @ 2020-11-11 12:51 UTC (permalink / raw)
Even though ALTER TABLE SET LOGGED/UNLOGGED does not require data
rewriting, currently it runs heap rewrite which causes large amount of
file I/O. This patch makes the command run without heap rewrite.
Addition to that, SET LOGGED while wal_level > minimal emits WAL using
XLOG_FPI instead of massive number of HEAP_INSERT's, which should be
smaller.
Also this allows for the cleanup of files left behind in the crash of
the transaction that created it.
---
src/backend/access/rmgrdesc/smgrdesc.c | 23 ++
src/backend/access/transam/README | 10 +
src/backend/catalog/storage.c | 420 +++++++++++++++++++++++--
src/backend/commands/tablecmds.c | 246 ++++++++++++---
src/backend/storage/buffer/bufmgr.c | 88 ++++++
src/backend/storage/file/reinit.c | 162 ++++++----
src/backend/storage/smgr/md.c | 4 +-
src/backend/storage/smgr/smgr.c | 6 +
src/common/relpath.c | 3 +-
src/include/catalog/storage.h | 2 +
src/include/catalog/storage_xlog.h | 22 +-
src/include/common/relpath.h | 5 +-
src/include/storage/bufmgr.h | 2 +
src/include/storage/smgr.h | 1 +
14 files changed, 854 insertions(+), 140 deletions(-)
diff --git a/src/backend/access/rmgrdesc/smgrdesc.c b/src/backend/access/rmgrdesc/smgrdesc.c
index 7755553d57..2c109b8ca4 100644
--- a/src/backend/access/rmgrdesc/smgrdesc.c
+++ b/src/backend/access/rmgrdesc/smgrdesc.c
@@ -40,6 +40,23 @@ smgr_desc(StringInfo buf, XLogReaderState *record)
xlrec->blkno, xlrec->flags);
pfree(path);
}
+ else if (info == XLOG_SMGR_UNLINK)
+ {
+ xl_smgr_unlink *xlrec = (xl_smgr_unlink *) rec;
+ char *path = relpathperm(xlrec->rnode, xlrec->forkNum);
+
+ appendStringInfoString(buf, path);
+ pfree(path);
+ }
+ else if (info == XLOG_SMGR_BUFPERSISTENCE)
+ {
+ xl_smgr_bufpersistence *xlrec = (xl_smgr_bufpersistence *) rec;
+ char *path = relpathperm(xlrec->rnode, MAIN_FORKNUM);
+
+ appendStringInfoString(buf, path);
+ appendStringInfo(buf, " persistence %d", xlrec->persistence);
+ pfree(path);
+ }
}
const char *
@@ -55,6 +72,12 @@ smgr_identify(uint8 info)
case XLOG_SMGR_TRUNCATE:
id = "TRUNCATE";
break;
+ case XLOG_SMGR_UNLINK:
+ id = "UNLINK";
+ break;
+ case XLOG_SMGR_BUFPERSISTENCE:
+ id = "BUFPERSISTENCE";
+ break;
}
return id;
diff --git a/src/backend/access/transam/README b/src/backend/access/transam/README
index 1edc8180c1..51616b2458 100644
--- a/src/backend/access/transam/README
+++ b/src/backend/access/transam/README
@@ -724,6 +724,16 @@ we must panic and abort recovery. The DBA will have to manually clean up
then restart recovery. This is part of the reason for not writing a WAL
entry until we've successfully done the original action.
+The INITTMP fork file
+--------------------------------
+
+An INITTMP fork is created when new relation file is created to mark
+the relfilenode needs to be cleaned up at recovery time. The file is
+removed at transaction end but is left when the process crashes before
+the transaction ends. In contrast to 4 above, failure to remove an
+INITTMP file will lead to data loss, in which case the server will
+shut down.
+
Skipping WAL for New RelFileNode
--------------------------------
diff --git a/src/backend/catalog/storage.c b/src/backend/catalog/storage.c
index cba7a9ada0..bd9680583b 100644
--- a/src/backend/catalog/storage.c
+++ b/src/backend/catalog/storage.c
@@ -19,6 +19,7 @@
#include "postgres.h"
+#include "access/amapi.h"
#include "access/parallel.h"
#include "access/visibilitymap.h"
#include "access/xact.h"
@@ -27,6 +28,7 @@
#include "access/xlogutils.h"
#include "catalog/storage.h"
#include "catalog/storage_xlog.h"
+#include "common/hashfn.h"
#include "miscadmin.h"
#include "storage/freespace.h"
#include "storage/smgr.h"
@@ -57,9 +59,16 @@ int wal_skip_threshold = 2048; /* in kilobytes */
* but I'm being paranoid.
*/
+#define PDOP_DELETE (0)
+#define PDOP_UNLINK_FORK (1 << 0)
+#define PDOP_SET_PERSISTENCE (1 << 1)
+
typedef struct PendingRelDelete
{
RelFileNode relnode; /* relation that may need to be deleted */
+ int op; /* operation mask */
+ bool bufpersistence; /* buffer persistence to set */
+ int unlink_forknum; /* forknum to unlink */
BackendId backend; /* InvalidBackendId if not a temp rel */
bool atCommit; /* T=delete at commit; F=delete at abort */
int nestLevel; /* xact nesting level of request */
@@ -75,6 +84,24 @@ typedef struct PendingRelSync
static PendingRelDelete *pendingDeletes = NULL; /* head of linked list */
HTAB *pendingSyncHash = NULL;
+typedef struct SRelHashEntry
+{
+ SMgrRelation srel;
+ char status; /* for simplehash use */
+} SRelHashEntry;
+
+/* define hashtable for workarea for pending deletes */
+#define SH_PREFIX srelhash
+#define SH_ELEMENT_TYPE SRelHashEntry
+#define SH_KEY_TYPE SMgrRelation
+#define SH_KEY srel
+#define SH_HASH_KEY(tb, key) \
+ hash_bytes((unsigned char *)&key, sizeof(SMgrRelation))
+#define SH_EQUAL(tb, a, b) (memcmp(&a, &b, sizeof(SMgrRelation)) == 0)
+#define SH_SCOPE static inline
+#define SH_DEFINE
+#define SH_DECLARE
+#include "lib/simplehash.h"
/*
* AddPendingSync
@@ -143,7 +170,17 @@ RelationCreateStorage(RelFileNode rnode, char relpersistence)
return NULL; /* placate compiler */
}
+ /*
+ * We are going to create a new storage file. If server crashes before the
+ * current transaction ends the file needs to be cleaned up but there's no
+ * clue to the orphan files. The inittmp fork works as the sentinel to
+ * identify that situation.
+ */
srel = smgropen(rnode, backend);
+ smgrcreate(srel, INITTMP_FORKNUM, false);
+ log_smgrcreate(&rnode, INITTMP_FORKNUM);
+ smgrimmedsync(srel, INITTMP_FORKNUM);
+
smgrcreate(srel, MAIN_FORKNUM, false);
if (needs_wal)
@@ -153,12 +190,25 @@ RelationCreateStorage(RelFileNode rnode, char relpersistence)
pending = (PendingRelDelete *)
MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete));
pending->relnode = rnode;
+ pending->op = PDOP_DELETE;
pending->backend = backend;
pending->atCommit = false; /* delete if abort */
pending->nestLevel = GetCurrentTransactionNestLevel();
pending->next = pendingDeletes;
pendingDeletes = pending;
+ /* drop inittmp fork at commit */
+ pending = (PendingRelDelete *)
+ MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete));
+ pending->relnode = rnode;
+ pending->op = PDOP_UNLINK_FORK;
+ pending->unlink_forknum = INITTMP_FORKNUM;
+ pending->backend = backend;
+ pending->atCommit = true;
+ pending->nestLevel = GetCurrentTransactionNestLevel();
+ pending->next = pendingDeletes;
+ pendingDeletes = pending;
+
if (relpersistence == RELPERSISTENCE_PERMANENT && !XLogIsNeeded())
{
Assert(backend == InvalidBackendId);
@@ -168,6 +218,215 @@ RelationCreateStorage(RelFileNode rnode, char relpersistence)
return srel;
}
+/*
+ * RelationCreateInitFork
+ * Create physical storage for the init fork of a relation.
+ *
+ * Create the init fork for the relation.
+ *
+ * This function is transactional. The creation is WAL-logged, and if the
+ * transaction aborts later on, the init fork will be removed.
+ */
+void
+RelationCreateInitFork(Relation rel)
+{
+ RelFileNode rnode = rel->rd_node;
+ PendingRelDelete *pending;
+ SMgrRelation srel;
+ PendingRelDelete *prev;
+ PendingRelDelete *next;
+ bool create = true;
+
+ /* switch buffer persistence */
+ SetRelationBuffersPersistence(rel->rd_smgr, false, false);
+
+ /*
+ * If we have entries for init-fork operation of this relation, that means
+ * that we have already registered pending sync entries to drop preexisting
+ * init fork since before the current transaction started. This function
+ * reverts that change just by removing the entries.
+ */
+ prev = NULL;
+ for (pending = pendingDeletes; pending != NULL; pending = next)
+ {
+ next = pending->next;
+ if (RelFileNodeEquals(rnode, pending->relnode) &&
+ pending->op != PDOP_DELETE)
+ {
+ if (prev)
+ prev->next = next;
+ else
+ pendingDeletes = next;
+ pfree(pending);
+
+ create = false;
+ }
+ else
+ {
+ /* unrelated entry, don't touch it */
+ prev = pending;
+ }
+ }
+
+ if (!create)
+ return;
+
+ /*
+ * We are going to create the init fork. If server crashes before the
+ * current transaction ends the init fork left alone corrupts data while
+ * recovery. The inittmp fork works as the sentinel to identify that
+ * situation.
+ */
+ srel = smgropen(rnode, InvalidBackendId);
+ smgrcreate(srel, INITTMP_FORKNUM, false);
+ log_smgrcreate(&rnode, INITTMP_FORKNUM);
+ smgrimmedsync(srel, INITTMP_FORKNUM);
+
+ /* We don't have existing init fork, create it. */
+ smgrcreate(srel, INIT_FORKNUM, false);
+
+ /*
+ * index-init fork needs further initialization. ambuildempty shoud do
+ * WAL-log and file sync by itself but otherwise we do that by myself.
+ */
+ if (rel->rd_rel->relkind == RELKIND_INDEX)
+ rel->rd_indam->ambuildempty(rel);
+ else
+ {
+ log_smgrcreate(&rnode, INIT_FORKNUM);
+ smgrimmedsync(srel, INIT_FORKNUM);
+ }
+
+ /* drop this init fork file at abort and revert persistence */
+ pending = (PendingRelDelete *)
+ MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete));
+ pending->relnode = rnode;
+ pending->op = PDOP_UNLINK_FORK | PDOP_SET_PERSISTENCE;
+ pending->unlink_forknum = INIT_FORKNUM;
+ pending->bufpersistence = true;
+ pending->backend = InvalidBackendId;
+ pending->atCommit = false;
+ pending->nestLevel = GetCurrentTransactionNestLevel();
+ pending->next = pendingDeletes;
+ pendingDeletes = pending;
+
+ /* drop inittmp fork at abort */
+ pending = (PendingRelDelete *)
+ MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete));
+ pending->relnode = rnode;
+ pending->op = PDOP_UNLINK_FORK;
+ pending->unlink_forknum = INITTMP_FORKNUM;
+ pending->backend = InvalidBackendId;
+ pending->atCommit = false;
+ pending->nestLevel = GetCurrentTransactionNestLevel();
+ pending->next = pendingDeletes;
+ pendingDeletes = pending;
+
+ /* drop inittmp fork at commit */
+ pending = (PendingRelDelete *)
+ MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete));
+ pending->relnode = rnode;
+ pending->op = PDOP_UNLINK_FORK;
+ pending->unlink_forknum = INITTMP_FORKNUM;
+ pending->backend = InvalidBackendId;
+ pending->atCommit = true;
+ pending->nestLevel = GetCurrentTransactionNestLevel();
+ pending->next = pendingDeletes;
+ pendingDeletes = pending;
+}
+
+/*
+ * RelationDropInitFork
+ * Delete physical storage for the init fork of a relation.
+ *
+ * Register pending-delete of the init fork. The real deletion is performed by
+ * smgrDoPendingDeletes at commit.
+ *
+ * This function is transactional. If the transaction aborts later on, the
+ * deletion doesn't happen.
+ */
+void
+RelationDropInitFork(Relation rel)
+{
+ RelFileNode rnode = rel->rd_node;
+ PendingRelDelete *pending;
+ PendingRelDelete *prev;
+ PendingRelDelete *next;
+ bool inxact_created = false;
+
+ /* switch buffer persistence */
+ SetRelationBuffersPersistence(rel->rd_smgr, true, false);
+
+ /*
+ * If we have entries for init-fork operation of this relation, that means
+ * that we have created the init fork in the current transaction. We
+ * immediately remove the init and inittmp forks immediately in that case.
+ * Otherwise just reister pending-delete for the existing init fork.
+ */
+ prev = NULL;
+ for (pending = pendingDeletes; pending != NULL; pending = next)
+ {
+ next = pending->next;
+ if (RelFileNodeEquals(rnode, pending->relnode) &&
+ pending->op != PDOP_DELETE)
+ {
+ /* unlink list entry */
+ if (prev)
+ prev->next = next;
+ else
+ pendingDeletes = next;
+ pfree(pending);
+
+ inxact_created = true;
+ }
+ else
+ {
+ /* unrelated entry, don't touch it */
+ prev = pending;
+ }
+ }
+
+ if (inxact_created)
+ {
+ SMgrRelation srel = smgropen(rnode, InvalidBackendId);
+
+ /*
+ * INIT/INITTMP forks never be loaded to shared buffer so no point in
+ * dropping buffers for these files.
+ */
+ log_smgrunlink(&rnode, INIT_FORKNUM);
+ smgrunlink(srel, INIT_FORKNUM, false);
+ log_smgrunlink(&rnode, INITTMP_FORKNUM);
+ smgrunlink(srel, INITTMP_FORKNUM, false);
+ smgrclose(srel);
+ return;
+ }
+
+ /* register drop of this init fork file at commit */
+ pending = (PendingRelDelete *)
+ MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete));
+ pending->relnode = rnode;
+ pending->op = PDOP_UNLINK_FORK;
+ pending->unlink_forknum = INIT_FORKNUM;
+ pending->backend = InvalidBackendId;
+ pending->atCommit = true;
+ pending->nestLevel = GetCurrentTransactionNestLevel();
+ pending->next = pendingDeletes;
+ pendingDeletes = pending;
+
+ /* revert buffer-persistence changes at abort */
+ pending = (PendingRelDelete *)
+ MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete));
+ pending->relnode = rnode;
+ pending->op = PDOP_SET_PERSISTENCE;
+ pending->bufpersistence = false;
+ pending->backend = InvalidBackendId;
+ pending->atCommit = true;
+ pending->nestLevel = GetCurrentTransactionNestLevel();
+ pending->next = pendingDeletes;
+ pendingDeletes = pending;
+}
+
/*
* Perform XLogInsert of an XLOG_SMGR_CREATE record to WAL.
*/
@@ -187,6 +446,44 @@ log_smgrcreate(const RelFileNode *rnode, ForkNumber forkNum)
XLogInsert(RM_SMGR_ID, XLOG_SMGR_CREATE | XLR_SPECIAL_REL_UPDATE);
}
+/*
+ * Perform XLogInsert of an XLOG_SMGR_UNLINK record to WAL.
+ */
+void
+log_smgrunlink(const RelFileNode *rnode, ForkNumber forkNum)
+{
+ xl_smgr_unlink xlrec;
+
+ /*
+ * Make an XLOG entry reporting the file unlink.
+ */
+ xlrec.rnode = *rnode;
+ xlrec.forkNum = forkNum;
+
+ XLogBeginInsert();
+ XLogRegisterData((char *) &xlrec, sizeof(xlrec));
+ XLogInsert(RM_SMGR_ID, XLOG_SMGR_UNLINK | XLR_SPECIAL_REL_UPDATE);
+}
+
+/*
+ * Perform XLogInsert of an XLOG_SMGR_BUFPERSISTENCE record to WAL.
+ */
+void
+log_smgrbufpersistence(const RelFileNode *rnode, bool persistence)
+{
+ xl_smgr_bufpersistence xlrec;
+
+ /*
+ * Make an XLOG entry reporting the change of buffer persistence.
+ */
+ xlrec.rnode = *rnode;
+ xlrec.persistence = persistence;
+
+ XLogBeginInsert();
+ XLogRegisterData((char *) &xlrec, sizeof(xlrec));
+ XLogInsert(RM_SMGR_ID, XLOG_SMGR_BUFPERSISTENCE | XLR_SPECIAL_REL_UPDATE);
+}
+
/*
* RelationDropStorage
* Schedule unlinking of physical storage at transaction commit.
@@ -200,6 +497,7 @@ RelationDropStorage(Relation rel)
pending = (PendingRelDelete *)
MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete));
pending->relnode = rel->rd_node;
+ pending->op = PDOP_DELETE;
pending->backend = rel->rd_backend;
pending->atCommit = true; /* delete if commit */
pending->nestLevel = GetCurrentTransactionNestLevel();
@@ -602,59 +900,97 @@ smgrDoPendingDeletes(bool isCommit)
int nrels = 0,
maxrels = 0;
SMgrRelation *srels = NULL;
+ srelhash_hash *close_srels = NULL;
+ bool found;
+
prev = NULL;
for (pending = pendingDeletes; pending != NULL; pending = next)
{
+ SMgrRelation srel;
+
next = pending->next;
if (pending->nestLevel < nestLevel)
{
/* outer-level entries should not be processed yet */
prev = pending;
+ continue;
}
+
+ /* unlink list entry first, so we don't retry on failure */
+ if (prev)
+ prev->next = next;
else
+ pendingDeletes = next;
+
+ if (pending->atCommit != isCommit)
{
- /* unlink list entry first, so we don't retry on failure */
- if (prev)
- prev->next = next;
- else
- pendingDeletes = next;
- /* do deletion if called for */
- if (pending->atCommit == isCommit)
- {
- SMgrRelation srel;
-
- srel = smgropen(pending->relnode, pending->backend);
-
- /* allocate the initial array, or extend it, if needed */
- if (maxrels == 0)
- {
- maxrels = 8;
- srels = palloc(sizeof(SMgrRelation) * maxrels);
- }
- else if (maxrels <= nrels)
- {
- maxrels *= 2;
- srels = repalloc(srels, sizeof(SMgrRelation) * maxrels);
- }
-
- srels[nrels++] = srel;
- }
/* must explicitly free the list entry */
pfree(pending);
/* prev does not change */
+ continue;
+ }
+
+ if (close_srels == NULL)
+ close_srels = srelhash_create(CurrentMemoryContext, 32, NULL);
+
+ srel = smgropen(pending->relnode, pending->backend);
+
+ /* Uniquify the smgr relations */
+ srelhash_insert(close_srels, srel, &found);
+
+ if (pending->op != PDOP_DELETE)
+ {
+ if (pending->op & PDOP_UNLINK_FORK)
+ {
+ /* other forks needs to drop buffers */
+ Assert(pending->unlink_forknum == INIT_FORKNUM ||
+ pending->unlink_forknum == INITTMP_FORKNUM);
+
+ log_smgrunlink(&pending->relnode, pending->unlink_forknum);
+ smgrunlink(srel, pending->unlink_forknum, false);
+
+ }
+
+ if (pending->op & PDOP_SET_PERSISTENCE)
+ SetRelationBuffersPersistence(srel, pending->bufpersistence,
+ false);
+ }
+ else
+ {
+ /* allocate the initial array, or extend it, if needed */
+ if (maxrels == 0)
+ {
+ maxrels = 8;
+ srels = palloc(sizeof(SMgrRelation) * maxrels);
+ }
+ else if (maxrels <= nrels)
+ {
+ maxrels *= 2;
+ srels = repalloc(srels, sizeof(SMgrRelation) * maxrels);
+ }
+
+ srels[nrels++] = srel;
}
}
if (nrels > 0)
{
smgrdounlinkall(srels, nrels, false);
-
- for (int i = 0; i < nrels; i++)
- smgrclose(srels[i]);
-
pfree(srels);
}
+
+ if (close_srels)
+ {
+ srelhash_iterator i;
+ SRelHashEntry *ent;
+
+ /* close smgr relatoins */
+ srelhash_start_iterate(close_srels, &i);
+ while ((ent = srelhash_iterate(close_srels, &i)) != NULL)
+ smgrclose(ent->srel);
+ srelhash_destroy(close_srels);
+ }
}
/*
@@ -824,7 +1160,8 @@ smgrGetPendingDeletes(bool forCommit, RelFileNode **ptr)
for (pending = pendingDeletes; pending != NULL; pending = pending->next)
{
if (pending->nestLevel >= nestLevel && pending->atCommit == forCommit
- && pending->backend == InvalidBackendId)
+ && pending->backend == InvalidBackendId
+ && pending->op == PDOP_DELETE)
nrels++;
}
if (nrels == 0)
@@ -837,7 +1174,8 @@ smgrGetPendingDeletes(bool forCommit, RelFileNode **ptr)
for (pending = pendingDeletes; pending != NULL; pending = pending->next)
{
if (pending->nestLevel >= nestLevel && pending->atCommit == forCommit
- && pending->backend == InvalidBackendId)
+ && pending->backend == InvalidBackendId &&
+ pending->op == PDOP_DELETE)
{
*rptr = pending->relnode;
rptr++;
@@ -917,6 +1255,15 @@ smgr_redo(XLogReaderState *record)
reln = smgropen(xlrec->rnode, InvalidBackendId);
smgrcreate(reln, xlrec->forkNum, true);
}
+ else if (info == XLOG_SMGR_UNLINK)
+ {
+ xl_smgr_unlink *xlrec = (xl_smgr_unlink *) XLogRecGetData(record);
+ SMgrRelation reln;
+
+ reln = smgropen(xlrec->rnode, InvalidBackendId);
+ smgrunlink(reln, xlrec->forkNum, true);
+ smgrclose(reln);
+ }
else if (info == XLOG_SMGR_TRUNCATE)
{
xl_smgr_truncate *xlrec = (xl_smgr_truncate *) XLogRecGetData(record);
@@ -1005,6 +1352,15 @@ smgr_redo(XLogReaderState *record)
FreeFakeRelcacheEntry(rel);
}
+ else if (info == XLOG_SMGR_BUFPERSISTENCE)
+ {
+ xl_smgr_bufpersistence *xlrec =
+ (xl_smgr_bufpersistence *) XLogRecGetData(record);
+ SMgrRelation reln;
+
+ reln = smgropen(xlrec->rnode, InvalidBackendId);
+ SetRelationBuffersPersistence(reln, xlrec->persistence, true);
+ }
else
elog(PANIC, "smgr_redo: unknown op code %u", info);
}
diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c
index 993da56d43..37a15d31ee 100644
--- a/src/backend/commands/tablecmds.c
+++ b/src/backend/commands/tablecmds.c
@@ -50,6 +50,7 @@
#include "commands/defrem.h"
#include "commands/event_trigger.h"
#include "commands/policy.h"
+#include "commands/progress.h"
#include "commands/sequence.h"
#include "commands/tablecmds.h"
#include "commands/tablespace.h"
@@ -4917,6 +4918,170 @@ ATParseTransformCmd(List **wqueue, AlteredTableInfo *tab, Relation rel,
return newcmd;
}
+/*
+ * RelationChangePersistence: do in-place persistence change of a relation
+ */
+static void
+RelationChangePersistence(AlteredTableInfo *tab, char persistence,
+ LOCKMODE lockmode)
+{
+ Relation rel;
+ Relation classRel;
+ HeapTuple tuple,
+ newtuple;
+ Datum new_val[Natts_pg_class];
+ bool new_null[Natts_pg_class],
+ new_repl[Natts_pg_class];
+ int i;
+ List *relids;
+ ListCell *lc_oid;
+
+ Assert(tab->rewrite == AT_REWRITE_ALTER_PERSISTENCE);
+ Assert(lockmode == AccessExclusiveLock);
+
+ /*
+ * Under the following condition, we need to call ATRewriteTable, which
+ * cannot be false in the AT_REWRITE_ALTER_PERSISTENCE case.
+ */
+ Assert(tab->constraints == NULL && tab->partition_constraint == NULL &&
+ tab->newvals == NULL && !tab->verify_new_notnull);
+
+ rel = table_open(tab->relid, lockmode);
+
+ Assert(rel->rd_rel->relpersistence != persistence);
+
+ elog(DEBUG1, "perform im-place persistnce change");
+
+ RelationOpenSmgr(rel);
+
+ /*
+ * First we collect all relations that we need to change persistence.
+ */
+
+ /* Collect OIDs of indexes and toast relations */
+ relids = RelationGetIndexList(rel);
+ relids = lcons_oid(rel->rd_id, relids);
+
+ /* Add toast relation if any */
+ if (OidIsValid(rel->rd_rel->reltoastrelid))
+ {
+ List *toastidx;
+ Relation toastrel = table_open(rel->rd_rel->reltoastrelid, lockmode);
+
+ RelationOpenSmgr(toastrel);
+ relids = lappend_oid(relids, rel->rd_rel->reltoastrelid);
+ toastidx = RelationGetIndexList(toastrel);
+ relids = list_concat(relids, toastidx);
+ pfree(toastidx);
+ table_close(toastrel, NoLock);
+ }
+
+ table_close(rel, NoLock);
+
+ /* Make changes in storage */
+ classRel = table_open(RelationRelationId, RowExclusiveLock);
+
+ foreach (lc_oid, relids)
+ {
+ Oid reloid = lfirst_oid(lc_oid);
+ Relation r = relation_open(reloid, lockmode);
+
+ /*
+ * Some access methods do not accept in-place persistence change. For
+ * example, GiST uses page LSNs to figure out whether a block has
+ * changed, where UNLOGGED GiST indexes use fake LSNs that are
+ * incompatible with real LSNs used for LOGGED ones.
+ *
+ * XXXX: We don't bother allowing in-place persistence change for index
+ * methods other than btree for now.
+ */
+ if (r->rd_rel->relkind == RELKIND_INDEX &&
+ r->rd_rel->relam != BTREE_AM_OID)
+ {
+ int reindex_flags;
+
+ /* reindex doesn't allow concurrent use of the index */
+ table_close(r, NoLock);
+
+ reindex_flags =
+ REINDEX_REL_SUPPRESS_INDEX_USE |
+ REINDEX_REL_CHECK_CONSTRAINTS;
+
+ /* Set the same persistence with the parent relation. */
+ if (persistence == RELPERSISTENCE_UNLOGGED)
+ reindex_flags |= REINDEX_REL_FORCE_INDEXES_UNLOGGED;
+ else
+ reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
+
+ reindex_index(reloid, reindex_flags, persistence, 0);
+
+ continue;
+ }
+
+ RelationOpenSmgr(r);
+
+ /* Create or drop init fork */
+ if (persistence == RELPERSISTENCE_UNLOGGED)
+ RelationCreateInitFork(r);
+ else
+ RelationDropInitFork(r);
+
+ /*
+ * When this relation gets WAL-logged, immediately sync all files but
+ * initfork to establish the initial state on storage. Buffers have
+ * already flushed out by RelationCreate(Drop)InitFork called just
+ * above. Initfork should have been synced as needed.
+ */
+ if (persistence == RELPERSISTENCE_PERMANENT)
+ {
+ for (i = 0 ; i < INIT_FORKNUM ; i++)
+ {
+ if (smgrexists(r->rd_smgr, i))
+ smgrimmedsync(r->rd_smgr, i);
+ }
+ }
+
+ /* Update catalog */
+ tuple = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(reloid));
+ if (!HeapTupleIsValid(tuple))
+ elog(ERROR, "cache lookup failed for relation %u", reloid);
+
+ memset(new_val, 0, sizeof(new_val));
+ memset(new_null, false, sizeof(new_null));
+ memset(new_repl, false, sizeof(new_repl));
+
+ new_val[Anum_pg_class_relpersistence - 1] = CharGetDatum(persistence);
+ new_null[Anum_pg_class_relpersistence - 1] = false;
+ new_repl[Anum_pg_class_relpersistence - 1] = true;
+
+ newtuple = heap_modify_tuple(tuple, RelationGetDescr(classRel),
+ new_val, new_null, new_repl);
+
+ CatalogTupleUpdate(classRel, &newtuple->t_self, newtuple);
+ heap_freetuple(newtuple);
+
+ /*
+ * While wal_level >= replica, switching to LOGGED requires the
+ * relation content to be WAL-logged to recover the table.
+ */
+ if (persistence == RELPERSISTENCE_PERMANENT && XLogIsNeeded())
+ {
+ ForkNumber fork;
+
+ for (fork = 0; fork < INIT_FORKNUM ; fork++)
+ {
+ if (smgrexists(r->rd_smgr, fork))
+ log_newpage_range(r, fork,
+ 0, smgrnblocks(r->rd_smgr, fork), false);
+ }
+ }
+
+ table_close(r, NoLock);
+ }
+
+ table_close(classRel, NoLock);
+}
+
/*
* ATRewriteTables: ALTER TABLE phase 3
*/
@@ -5037,45 +5202,52 @@ ATRewriteTables(AlterTableStmt *parsetree, List **wqueue, LOCKMODE lockmode,
tab->relid,
tab->rewrite);
- /*
- * Create transient table that will receive the modified data.
- *
- * Ensure it is marked correctly as logged or unlogged. We have
- * to do this here so that buffers for the new relfilenode will
- * have the right persistence set, and at the same time ensure
- * that the original filenode's buffers will get read in with the
- * correct setting (i.e. the original one). Otherwise a rollback
- * after the rewrite would possibly result with buffers for the
- * original filenode having the wrong persistence setting.
- *
- * NB: This relies on swap_relation_files() also swapping the
- * persistence. That wouldn't work for pg_class, but that can't be
- * unlogged anyway.
- */
- OIDNewHeap = make_new_heap(tab->relid, NewTableSpace, persistence,
- lockmode);
+ if (tab->rewrite == AT_REWRITE_ALTER_PERSISTENCE)
+ RelationChangePersistence(tab, persistence, lockmode);
+ else
+ {
+ /*
+ * Create transient table that will receive the modified data.
+ *
+ * Ensure it is marked correctly as logged or unlogged. We
+ * have to do this here so that buffers for the new relfilenode
+ * will have the right persistence set, and at the same time
+ * ensure that the original filenode's buffers will get read in
+ * with the correct setting (i.e. the original one). Otherwise
+ * a rollback after the rewrite would possibly result with
+ * buffers for the original filenode having the wrong
+ * persistence setting.
+ *
+ * NB: This relies on swap_relation_files() also swapping the
+ * persistence. That wouldn't work for pg_class, but that can't
+ * be unlogged anyway.
+ */
+ OIDNewHeap = make_new_heap(tab->relid, NewTableSpace, persistence,
+ lockmode);
- /*
- * Copy the heap data into the new table with the desired
- * modifications, and test the current data within the table
- * against new constraints generated by ALTER TABLE commands.
- */
- ATRewriteTable(tab, OIDNewHeap, lockmode);
+ /*
+ * Copy the heap data into the new table with the desired
+ * modifications, and test the current data within the table
+ * against new constraints generated by ALTER TABLE commands.
+ */
+ ATRewriteTable(tab, OIDNewHeap, lockmode);
- /*
- * Swap the physical files of the old and new heaps, then rebuild
- * indexes and discard the old heap. We can use RecentXmin for
- * the table's new relfrozenxid because we rewrote all the tuples
- * in ATRewriteTable, so no older Xid remains in the table. Also,
- * we never try to swap toast tables by content, since we have no
- * interest in letting this code work on system catalogs.
- */
- finish_heap_swap(tab->relid, OIDNewHeap,
- false, false, true,
- !OidIsValid(tab->newTableSpace),
- RecentXmin,
- ReadNextMultiXactId(),
- persistence);
+ /*
+ * Swap the physical files of the old and new heaps, then
+ * rebuild indexes and discard the old heap. We can use
+ * RecentXmin for the table's new relfrozenxid because we
+ * rewrote all the tuples in ATRewriteTable, so no older Xid
+ * remains in the table. Also, we never try to swap toast
+ * tables by content, since we have no interest in letting this
+ * code work on system catalogs.
+ */
+ finish_heap_swap(tab->relid, OIDNewHeap,
+ false, false, true,
+ !OidIsValid(tab->newTableSpace),
+ RecentXmin,
+ ReadNextMultiXactId(),
+ persistence);
+ }
}
else
{
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index 71b5852224..b730b4417c 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -37,6 +37,7 @@
#include "access/xlog.h"
#include "catalog/catalog.h"
#include "catalog/storage.h"
+#include "catalog/storage_xlog.h"
#include "executor/instrument.h"
#include "lib/binaryheap.h"
#include "miscadmin.h"
@@ -3032,6 +3033,93 @@ DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber *forkNum,
}
}
+/* ---------------------------------------------------------------------
+ * SetRelFileNodeBuffersPersistence
+ *
+ * This function changes the persistence of all buffer pages of a relation
+ * then writes all dirty pages of the relation out to disk when switching
+ * to PERMANENT. (or more accurately, out to kernel disk buffers),
+ * ensuring that the kernel has an up-to-date view of the relation.
+ *
+ * Generally, the caller should be holding AccessExclusiveLock on the
+ * target relation to ensure that no other backend is busy dirtying
+ * more blocks of the relation; the effects can't be expected to last
+ * after the lock is released.
+ *
+ * XXX currently it sequentially searches the buffer pool, should be
+ * changed to more clever ways of searching. This routine is not
+ * used in any performance-critical code paths, so it's not worth
+ * adding additional overhead to normal paths to make it go faster;
+ * but see also DropRelFileNodeBuffers.
+ * --------------------------------------------------------------------
+ */
+void
+SetRelationBuffersPersistence(SMgrRelation srel, bool permanent, bool isRedo)
+{
+ int i;
+ RelFileNodeBackend rnode = srel->smgr_rnode;
+
+ Assert (!RelFileNodeBackendIsTemp(rnode));
+
+ if (!isRedo)
+ log_smgrbufpersistence(&srel->smgr_rnode.node, permanent);
+
+ ResourceOwnerEnlargeBuffers(CurrentResourceOwner);
+
+ for (i = 0; i < NBuffers; i++)
+ {
+ BufferDesc *bufHdr = GetBufferDescriptor(i);
+ uint32 buf_state;
+
+ if (!RelFileNodeEquals(bufHdr->tag.rnode, rnode.node))
+ continue;
+
+ ReservePrivateRefCountEntry();
+
+ buf_state = LockBufHdr(bufHdr);
+
+ if (!RelFileNodeEquals(bufHdr->tag.rnode, rnode.node))
+ {
+ UnlockBufHdr(bufHdr, buf_state);
+ continue;
+ }
+
+ if (permanent)
+ {
+ /* Init fork is being dropped, drop buffers for it. */
+ if (bufHdr->tag.forkNum == INIT_FORKNUM)
+ {
+ InvalidateBuffer(bufHdr);
+ continue;
+ }
+
+ buf_state |= BM_PERMANENT;
+ pg_atomic_write_u32(&bufHdr->state, buf_state);
+
+ /* we flush this buffer when switching to PERMANENT */
+ if ((buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY))
+ {
+ PinBuffer_Locked(bufHdr);
+ LWLockAcquire(BufferDescriptorGetContentLock(bufHdr),
+ LW_SHARED);
+ FlushBuffer(bufHdr, srel);
+ LWLockRelease(BufferDescriptorGetContentLock(bufHdr));
+ UnpinBuffer(bufHdr, true);
+ }
+ else
+ UnlockBufHdr(bufHdr, buf_state);
+ }
+ else
+ {
+ /* init fork is always BM_PERMANENT. See BufferAlloc */
+ if (bufHdr->tag.forkNum != INIT_FORKNUM)
+ buf_state &= ~BM_PERMANENT;
+
+ UnlockBufHdr(bufHdr, buf_state);
+ }
+ }
+}
+
/* ---------------------------------------------------------------------
* DropRelFileNodesAllBuffers
*
diff --git a/src/backend/storage/file/reinit.c b/src/backend/storage/file/reinit.c
index 40c758d789..adcb54b0fa 100644
--- a/src/backend/storage/file/reinit.c
+++ b/src/backend/storage/file/reinit.c
@@ -31,7 +31,8 @@ static void ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname,
typedef struct
{
Oid reloid; /* hash key */
-} unlogged_relation_entry;
+ bool dirty; /* to be removed */
+} relfile_entry;
/*
* Reset unlogged relations from before the last restart.
@@ -151,6 +152,8 @@ ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname, int op)
DIR *dbspace_dir;
struct dirent *de;
char rm_path[MAXPGPATH * 2];
+ HTAB *hash;
+ HASHCTL ctl;
/* Caller must specify at least one operation. */
Assert((op & (UNLOGGED_RELATION_CLEANUP | UNLOGGED_RELATION_INIT)) != 0);
@@ -160,88 +163,86 @@ ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname, int op)
* the files with init forks. Then, we go through again and nuke
* everything with the same OID except the init fork.
*/
+
+ /*
+ * It's possible that someone could create a ton of unlogged relations
+ * in the same database & tablespace, so we'd better use a hash table
+ * rather than an array or linked list to keep track of which files
+ * need to be reset. Otherwise, this cleanup operation would be
+ * O(n^2).
+ */
+ memset(&ctl, 0, sizeof(ctl));
+ ctl.keysize = sizeof(Oid);
+ ctl.entrysize = sizeof(relfile_entry);
+ hash = hash_create("inittmp hash", 32, &ctl, HASH_ELEM | HASH_BLOBS);
+
+ /* Collect inttmp forks in the directory. */
+ dbspace_dir = AllocateDir(dbspacedirname);
+ while ((de = ReadDir(dbspace_dir, dbspacedirname)) != NULL)
+ {
+ int oidchars;
+ ForkNumber forkNum;
+
+ /* Skip anything that doesn't look like a relation data file. */
+ if (!parse_filename_for_nontemp_relation(de->d_name, &oidchars,
+ &forkNum))
+ continue;
+
+ /* Record init and inittmp forks */
+ if (forkNum == INIT_FORKNUM || forkNum == INITTMP_FORKNUM)
+ {
+ Oid key;
+ relfile_entry *ent;
+ bool found;
+
+ /*
+ * Record the relfilenode. If it has INITTMP fork, the all files
+ * needs to be cleaned up. Otherwise the relfilenode is cleaned up
+ * according to the unloggedness.
+ */
+ key = atooid(de->d_name);
+ ent = hash_search(hash, &key, HASH_ENTER, &found);
+
+ if (!found)
+ ent->dirty = false;
+
+ if (forkNum == INITTMP_FORKNUM)
+ ent->dirty = true;
+ }
+ }
+
+ /* Done with the first pass. */
+ FreeDir(dbspace_dir);
+
if ((op & UNLOGGED_RELATION_CLEANUP) != 0)
{
- HTAB *hash;
- HASHCTL ctl;
-
- /*
- * It's possible that someone could create a ton of unlogged relations
- * in the same database & tablespace, so we'd better use a hash table
- * rather than an array or linked list to keep track of which files
- * need to be reset. Otherwise, this cleanup operation would be
- * O(n^2).
- */
- ctl.keysize = sizeof(Oid);
- ctl.entrysize = sizeof(unlogged_relation_entry);
- ctl.hcxt = CurrentMemoryContext;
- hash = hash_create("unlogged relation OIDs", 32, &ctl,
- HASH_ELEM | HASH_BLOBS | HASH_CONTEXT);
-
- /* Scan the directory. */
- dbspace_dir = AllocateDir(dbspacedirname);
- while ((de = ReadDir(dbspace_dir, dbspacedirname)) != NULL)
- {
- ForkNumber forkNum;
- int oidchars;
- unlogged_relation_entry ent;
-
- /* Skip anything that doesn't look like a relation data file. */
- if (!parse_filename_for_nontemp_relation(de->d_name, &oidchars,
- &forkNum))
- continue;
-
- /* Also skip it unless this is the init fork. */
- if (forkNum != INIT_FORKNUM)
- continue;
-
- /*
- * Put the OID portion of the name into the hash table, if it
- * isn't already.
- */
- ent.reloid = atooid(de->d_name);
- (void) hash_search(hash, &ent, HASH_ENTER, NULL);
- }
-
- /* Done with the first pass. */
- FreeDir(dbspace_dir);
-
- /*
- * If we didn't find any init forks, there's no point in continuing;
- * we can bail out now.
- */
- if (hash_get_num_entries(hash) == 0)
- {
- hash_destroy(hash);
- return;
- }
-
/*
* Now, make a second pass and remove anything that matches.
*/
dbspace_dir = AllocateDir(dbspacedirname);
while ((de = ReadDir(dbspace_dir, dbspacedirname)) != NULL)
{
- ForkNumber forkNum;
- int oidchars;
- unlogged_relation_entry ent;
+ ForkNumber forkNum;
+ int oidchars;
+ Oid key;
+ relfile_entry *ent;
/* Skip anything that doesn't look like a relation data file. */
if (!parse_filename_for_nontemp_relation(de->d_name, &oidchars,
&forkNum))
continue;
- /* We never remove the init fork. */
- if (forkNum == INIT_FORKNUM)
- continue;
-
/*
* See whether the OID portion of the name shows up in the hash
* table. If so, nuke it!
*/
- ent.reloid = atooid(de->d_name);
- if (hash_search(hash, &ent, HASH_FIND, NULL))
+ key = atooid(de->d_name);
+ ent = hash_search(hash, &key, HASH_FIND, NULL);
+
+ /* we don't remove clean init file */
+ if (ent && (ent->dirty || forkNum != INIT_FORKNUM))
{
+ /* so, nuke it! */
snprintf(rm_path, sizeof(rm_path), "%s/%s",
dbspacedirname, de->d_name);
if (unlink(rm_path) < 0)
@@ -256,7 +257,6 @@ ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname, int op)
/* Cleanup is complete. */
FreeDir(dbspace_dir);
- hash_destroy(hash);
}
/*
@@ -277,12 +277,42 @@ ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname, int op)
char oidbuf[OIDCHARS + 1];
char srcpath[MAXPGPATH * 2];
char dstpath[MAXPGPATH];
+ Oid key;
+ relfile_entry *ent;
/* Skip anything that doesn't look like a relation data file. */
if (!parse_filename_for_nontemp_relation(de->d_name, &oidchars,
&forkNum))
continue;
+ /*
+ * See whether the OID portion of the name shows up in the hash
+ * table.
+ */
+ key = atooid(de->d_name);
+ ent = hash_search(hash, &key, HASH_FIND, NULL);
+
+ /* we don't remove clean init file */
+ if (ent && (ent->dirty || forkNum != INIT_FORKNUM))
+ {
+ /*
+ * The file is dirty. It shoudl have been removed once at
+ * cleanup time but recovery can create them again. Remove
+ * them.
+ */
+ snprintf(rm_path, sizeof(rm_path), "%s/%s",
+ dbspacedirname, de->d_name);
+ if (unlink(rm_path) < 0)
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not remove file \"%s\": %m",
+ rm_path)));
+ else
+ elog(DEBUG2, "unlinked file \"%s\"", rm_path);
+
+ continue;
+ }
+
/* Also skip it unless this is the init fork. */
if (forkNum != INIT_FORKNUM)
continue;
@@ -351,6 +381,8 @@ ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname, int op)
*/
fsync_fname(dbspacedirname, true);
}
+
+ hash_destroy(hash);
}
/*
diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c
index 0643d714fb..416fd859e6 100644
--- a/src/backend/storage/smgr/md.c
+++ b/src/backend/storage/smgr/md.c
@@ -338,8 +338,10 @@ mdunlinkfork(RelFileNodeBackend rnode, ForkNumber forkNum, bool isRedo)
if (ret == 0 || errno != ENOENT)
{
ret = unlink(path);
+
+ /* failure of removing inittmp fork leads to a data loss. */
if (ret < 0 && errno != ENOENT)
- ereport(WARNING,
+ ereport((forkNum != INITTMP_FORKNUM ? WARNING : ERROR),
(errcode_for_file_access(),
errmsg("could not remove file \"%s\": %m", path)));
}
diff --git a/src/backend/storage/smgr/smgr.c b/src/backend/storage/smgr/smgr.c
index 0f31ff3822..4102d3d59c 100644
--- a/src/backend/storage/smgr/smgr.c
+++ b/src/backend/storage/smgr/smgr.c
@@ -644,6 +644,12 @@ smgrimmedsync(SMgrRelation reln, ForkNumber forknum)
smgrsw[reln->smgr_which].smgr_immedsync(reln, forknum);
}
+void
+smgrunlink(SMgrRelation reln, ForkNumber forknum, bool isRedo)
+{
+ smgrsw[reln->smgr_which].smgr_unlink(reln->smgr_rnode, forknum, isRedo);
+}
+
/*
* AtEOXact_SMgr
*
diff --git a/src/common/relpath.c b/src/common/relpath.c
index 1f5c426ec0..2954cd9c24 100644
--- a/src/common/relpath.c
+++ b/src/common/relpath.c
@@ -34,7 +34,8 @@ const char *const forkNames[] = {
"main", /* MAIN_FORKNUM */
"fsm", /* FSM_FORKNUM */
"vm", /* VISIBILITYMAP_FORKNUM */
- "init" /* INIT_FORKNUM */
+ "init", /* INIT_FORKNUM */
+ "itmp" /* INITTMP_FORKNUM */
};
StaticAssertDecl(lengthof(forkNames) == (MAX_FORKNUM + 1),
diff --git a/src/include/catalog/storage.h b/src/include/catalog/storage.h
index 0ab32b44e9..382623159c 100644
--- a/src/include/catalog/storage.h
+++ b/src/include/catalog/storage.h
@@ -23,6 +23,8 @@
extern int wal_skip_threshold;
extern SMgrRelation RelationCreateStorage(RelFileNode rnode, char relpersistence);
+extern void RelationCreateInitFork(Relation rel);
+extern void RelationDropInitFork(Relation rel);
extern void RelationDropStorage(Relation rel);
extern void RelationPreserveStorage(RelFileNode rnode, bool atCommit);
extern void RelationPreTruncate(Relation rel);
diff --git a/src/include/catalog/storage_xlog.h b/src/include/catalog/storage_xlog.h
index f0814f1458..0fd0832a8b 100644
--- a/src/include/catalog/storage_xlog.h
+++ b/src/include/catalog/storage_xlog.h
@@ -22,13 +22,17 @@
/*
* Declarations for smgr-related XLOG records
*
- * Note: we log file creation and truncation here, but logging of deletion
- * actions is handled by xact.c, because it is part of transaction commit.
+ * Note: we log file creation, truncation, deletion and persistence change
+ * here. logging of deletion actions is mainly handled by xact.c, because it is
+ * part of transaction commit, but we log deletions happens outside of a
+ * transaction.
*/
/* XLOG gives us high 4 bits */
#define XLOG_SMGR_CREATE 0x10
#define XLOG_SMGR_TRUNCATE 0x20
+#define XLOG_SMGR_UNLINK 0x30
+#define XLOG_SMGR_BUFPERSISTENCE 0x40
typedef struct xl_smgr_create
{
@@ -36,6 +40,18 @@ typedef struct xl_smgr_create
ForkNumber forkNum;
} xl_smgr_create;
+typedef struct xl_smgr_unlink
+{
+ RelFileNode rnode;
+ ForkNumber forkNum;
+} xl_smgr_unlink;
+
+typedef struct xl_smgr_bufpersistence
+{
+ RelFileNode rnode;
+ bool persistence;
+} xl_smgr_bufpersistence;
+
/* flags for xl_smgr_truncate */
#define SMGR_TRUNCATE_HEAP 0x0001
#define SMGR_TRUNCATE_VM 0x0002
@@ -51,6 +67,8 @@ typedef struct xl_smgr_truncate
} xl_smgr_truncate;
extern void log_smgrcreate(const RelFileNode *rnode, ForkNumber forkNum);
+extern void log_smgrunlink(const RelFileNode *rnode, ForkNumber forkNum);
+extern void log_smgrbufpersistence(const RelFileNode *rnode, bool persistence);
extern void smgr_redo(XLogReaderState *record);
extern void smgr_desc(StringInfo buf, XLogReaderState *record);
diff --git a/src/include/common/relpath.h b/src/include/common/relpath.h
index a44be11ca0..4305bdbe96 100644
--- a/src/include/common/relpath.h
+++ b/src/include/common/relpath.h
@@ -43,7 +43,8 @@ typedef enum ForkNumber
MAIN_FORKNUM = 0,
FSM_FORKNUM,
VISIBILITYMAP_FORKNUM,
- INIT_FORKNUM
+ INIT_FORKNUM,
+ INITTMP_FORKNUM
/*
* NOTE: if you add a new fork, change MAX_FORKNUM and possibly
@@ -52,7 +53,7 @@ typedef enum ForkNumber
*/
} ForkNumber;
-#define MAX_FORKNUM INIT_FORKNUM
+#define MAX_FORKNUM INITTMP_FORKNUM
#define FORKNAMECHARS 4 /* max chars for a fork name */
diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h
index ff6cd0fc54..d9752a8317 100644
--- a/src/include/storage/bufmgr.h
+++ b/src/include/storage/bufmgr.h
@@ -205,6 +205,8 @@ extern void FlushRelationsAllBuffers(struct SMgrRelationData **smgrs, int nrels)
extern void FlushDatabaseBuffers(Oid dbid);
extern void DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber *forkNum,
int nforks, BlockNumber *firstDelBlock);
+extern void SetRelationBuffersPersistence(struct SMgrRelationData *srel,
+ bool permanent, bool isRedo);
extern void DropRelFileNodesAllBuffers(RelFileNodeBackend *rnodes, int nnodes);
extern void DropDatabaseBuffers(Oid dbid);
diff --git a/src/include/storage/smgr.h b/src/include/storage/smgr.h
index ebf4a199dc..8be17d9afc 100644
--- a/src/include/storage/smgr.h
+++ b/src/include/storage/smgr.h
@@ -86,6 +86,7 @@ extern void smgrclose(SMgrRelation reln);
extern void smgrcloseall(void);
extern void smgrclosenode(RelFileNodeBackend rnode);
extern void smgrcreate(SMgrRelation reln, ForkNumber forknum, bool isRedo);
+extern void smgrunlink(SMgrRelation reln, ForkNumber forknum, bool isRedo);
extern void smgrdosyncall(SMgrRelation *rels, int nrels);
extern void smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo);
extern void smgrextend(SMgrRelation reln, ForkNumber forknum,
--
2.27.0
----Next_Part(Fri_Jan__8_14_47_05_2021_579)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v3-0002-New-command-ALTER-TABLE-ALL-IN-TABLESPACE-SET-LOG.patch"
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-04-27 06:12 Michael Paquier <[email protected]>
2 siblings, 0 replies; 58+ messages in thread
From: Michael Paquier @ 2022-04-27 06:12 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: Tomas Vondra <[email protected]>; Alvaro Herrera <[email protected]>; Pg Hackers <[email protected]>
On Wed, Apr 27, 2022 at 10:25:50AM +0530, Amit Kapila wrote:
> I feel we can explain a bit more about this in docs. We already have
> some explanation of how row filters are combined [1]. We can probably
> add a few examples for column lists.
I am not completely sure exactly what we should do here, but this
stuff needs to be at least discussed. I have added an open item.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../Ymje09ToDVKpW%[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-04-28 15:35 Tomas Vondra <[email protected]>
2 siblings, 1 reply; 58+ messages in thread
From: Tomas Vondra @ 2022-04-28 15:35 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Pg Hackers <[email protected]>
On 4/28/22 05:17, Amit Kapila wrote:
> On Thu, Apr 28, 2022 at 3:26 AM Tomas Vondra
> <[email protected]> wrote:
>>
>> so I've been looking at tweaking the code so that the behavior matches
>> Alvaro's expectations. It passes check-world but I'm not claiming it's
>> nowhere near commitable - the purpose is mostly to give better idea of
>> how invasive the change is etc.
>>
>
> I was just skimming through the patch and didn't find anything related
> to initial sync handling. I feel the behavior should be same for
> initial sync and replication.
>
Yeah, sorry for not mentioning that - my goal was to explore and try
getting the behavior in regular replication right first, before
attempting to do the same thing in tablesync.
Attached is a patch doing the same thing in tablesync. The overall idea
is to generate copy statement with CASE expressions, applying filters to
individual columns. For Alvaro's example, this generates something like
SELECT
(CASE WHEN (a < 0) OR (a > 0) THEN a ELSE NULL END) AS a,
(CASE WHEN (a > 0) THEN b ELSE NULL END) AS b,
(CASE WHEN (a < 0) THEN c ELSE NULL END) AS c
FROM uno WHERE (a < 0) OR (a > 0)
And that seems to work fine. Similarly to regular replication we have to
use both the "total" column list (union of per-publication lists) and
per-publication (row filter + column list), but that's expected.
There's a couple options how we might optimize this for common cases.
For example if there's just a single publication, there's no need to
generate the CASE expressions - the WHERE filter will do the trick.
regards
--
Tomas Vondra
EnterpriseDB: http://www.enterprisedb.com
The Enterprise PostgreSQL Company
Attachments:
[text/x-patch] rework-column-row-filtering-v2.patch (38.2K, ../../[email protected]/2-rework-column-row-filtering-v2.patch)
download | inline diff:
diff --git a/src/backend/replication/logical/proto.c b/src/backend/replication/logical/proto.c
index ff8513e2d29..41c5e3413f6 100644
--- a/src/backend/replication/logical/proto.c
+++ b/src/backend/replication/logical/proto.c
@@ -33,7 +33,9 @@ static void logicalrep_write_attrs(StringInfo out, Relation rel,
Bitmapset *columns);
static void logicalrep_write_tuple(StringInfo out, Relation rel,
TupleTableSlot *slot,
- bool binary, Bitmapset *columns);
+ bool binary,
+ Bitmapset *schema_columns,
+ Bitmapset *columns);
static void logicalrep_read_attrs(StringInfo in, LogicalRepRelation *rel);
static void logicalrep_read_tuple(StringInfo in, LogicalRepTupleData *tuple);
@@ -412,7 +414,8 @@ logicalrep_read_origin(StringInfo in, XLogRecPtr *origin_lsn)
*/
void
logicalrep_write_insert(StringInfo out, TransactionId xid, Relation rel,
- TupleTableSlot *newslot, bool binary, Bitmapset *columns)
+ TupleTableSlot *newslot, bool binary,
+ Bitmapset *schema_columns, Bitmapset *columns)
{
pq_sendbyte(out, LOGICAL_REP_MSG_INSERT);
@@ -424,7 +427,8 @@ logicalrep_write_insert(StringInfo out, TransactionId xid, Relation rel,
pq_sendint32(out, RelationGetRelid(rel));
pq_sendbyte(out, 'N'); /* new tuple follows */
- logicalrep_write_tuple(out, rel, newslot, binary, columns);
+ logicalrep_write_tuple(out, rel, newslot, binary,
+ schema_columns, columns);
}
/*
@@ -457,7 +461,8 @@ logicalrep_read_insert(StringInfo in, LogicalRepTupleData *newtup)
void
logicalrep_write_update(StringInfo out, TransactionId xid, Relation rel,
TupleTableSlot *oldslot, TupleTableSlot *newslot,
- bool binary, Bitmapset *columns)
+ bool binary, Bitmapset *schema_columns,
+ Bitmapset *columns)
{
pq_sendbyte(out, LOGICAL_REP_MSG_UPDATE);
@@ -478,11 +483,12 @@ logicalrep_write_update(StringInfo out, TransactionId xid, Relation rel,
pq_sendbyte(out, 'O'); /* old tuple follows */
else
pq_sendbyte(out, 'K'); /* old key follows */
- logicalrep_write_tuple(out, rel, oldslot, binary, NULL);
+ logicalrep_write_tuple(out, rel, oldslot, binary, NULL, NULL);
}
pq_sendbyte(out, 'N'); /* new tuple follows */
- logicalrep_write_tuple(out, rel, newslot, binary, columns);
+ logicalrep_write_tuple(out, rel, newslot, binary,
+ schema_columns, columns);
}
/*
@@ -551,7 +557,7 @@ logicalrep_write_delete(StringInfo out, TransactionId xid, Relation rel,
else
pq_sendbyte(out, 'K'); /* old key follows */
- logicalrep_write_tuple(out, rel, oldslot, binary, NULL);
+ logicalrep_write_tuple(out, rel, oldslot, binary, NULL, NULL);
}
/*
@@ -766,7 +772,8 @@ logicalrep_read_typ(StringInfo in, LogicalRepTyp *ltyp)
*/
static void
logicalrep_write_tuple(StringInfo out, Relation rel, TupleTableSlot *slot,
- bool binary, Bitmapset *columns)
+ bool binary,
+ Bitmapset *schema_columns, Bitmapset *columns)
{
TupleDesc desc;
Datum *values;
@@ -783,7 +790,7 @@ logicalrep_write_tuple(StringInfo out, Relation rel, TupleTableSlot *slot,
if (att->attisdropped || att->attgenerated)
continue;
- if (!column_in_column_list(att->attnum, columns))
+ if (!column_in_column_list(att->attnum, schema_columns))
continue;
nliveatts++;
@@ -804,10 +811,23 @@ logicalrep_write_tuple(StringInfo out, Relation rel, TupleTableSlot *slot,
if (att->attisdropped || att->attgenerated)
continue;
- if (!column_in_column_list(att->attnum, columns))
+ /*
+ * Columns that are not in schema (union of column lists) should
+ * be skipped entirely.
+ */
+ if (!column_in_column_list(att->attnum, schema_columns))
continue;
- if (isnull[i])
+ /*
+ * Columns not in the column list (derived consindering row filters)
+ * we just send NULL.
+ *
+ * XXX Not sure this is quite correct, though. Imagine you replicate
+ * values for columns (A,B), but it changes the row filter. Can we
+ * send NULL that would overwrite "proper" value replicated earlier?
+ */
+ if (isnull[i] ||
+ !column_in_column_list(att->attnum, columns))
{
pq_sendbyte(out, LOGICALREP_COLUMN_NULL);
continue;
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 49ceec3bdc8..fd547e16f4a 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -126,6 +126,20 @@ static bool FetchTableStates(bool *started_tx);
StringInfo copybuf = NULL;
+/*
+ * Info use to track and evaluate row filters for each publication the relation
+ * is included in, and calculat ethe column list.
+ */
+typedef struct PublicationInfo {
+
+ /* row filter (expression state) */
+ Node *rowfilter;
+
+ /* column list */
+ Bitmapset *columns;
+
+} PublicationInfo;
+
/*
* Exit routine for synchronization worker.
*/
@@ -696,14 +710,14 @@ copy_read_data(void *outbuf, int minread, int maxread)
*/
static void
fetch_remote_table_info(char *nspname, char *relname,
- LogicalRepRelation *lrel, List **qual)
+ LogicalRepRelation *lrel, List **pubinfos)
{
WalRcvExecResult *res;
StringInfoData cmd;
TupleTableSlot *slot;
Oid tableRow[] = {OIDOID, CHAROID, CHAROID};
Oid attrRow[] = {INT2OID, TEXTOID, OIDOID, BOOLOID};
- Oid qualRow[] = {TEXTOID};
+ Oid qualRow[] = {TEXTOID, INT2VECTOROID};
bool isnull;
int natt;
ListCell *lc;
@@ -878,6 +892,7 @@ fetch_remote_table_info(char *nspname, char *relname,
/* We don't know the number of rows coming, so allocate enough space. */
lrel->attnames = palloc0(MaxTupleAttributeNumber * sizeof(char *));
lrel->atttyps = palloc0(MaxTupleAttributeNumber * sizeof(Oid));
+ lrel->attnums = palloc0(MaxTupleAttributeNumber * sizeof(int16));
lrel->attkeys = NULL;
/*
@@ -905,6 +920,7 @@ fetch_remote_table_info(char *nspname, char *relname,
Assert(!isnull);
lrel->attnames[natt] = rel_colname;
+ lrel->attnums[natt] = attnum;
lrel->atttyps[natt] = DatumGetObjectId(slot_getattr(slot, 3, &isnull));
Assert(!isnull);
@@ -943,6 +959,7 @@ fetch_remote_table_info(char *nspname, char *relname,
* 3) one of the subscribed publications is declared as ALL TABLES IN
* SCHEMA that includes this relation
*/
+ *pubinfos = NIL;
if (walrcv_server_version(LogRepWorkerWalRcvConn) >= 150000)
{
StringInfoData pub_names;
@@ -965,7 +982,7 @@ fetch_remote_table_info(char *nspname, char *relname,
/* Check for row filters. */
resetStringInfo(&cmd);
appendStringInfo(&cmd,
- "SELECT DISTINCT pg_get_expr(pr.prqual, pr.prrelid)"
+ "SELECT DISTINCT pg_get_expr(pr.prqual, pr.prrelid), pr.prattrs"
" FROM pg_publication p"
" LEFT OUTER JOIN pg_publication_rel pr"
" ON (p.oid = pr.prpubid AND pr.prrelid = %u),"
@@ -976,7 +993,7 @@ fetch_remote_table_info(char *nspname, char *relname,
lrel->remoteid,
pub_names.data);
- res = walrcv_exec(LogRepWorkerWalRcvConn, cmd.data, 1, qualRow);
+ res = walrcv_exec(LogRepWorkerWalRcvConn, cmd.data, lengthof(qualRow), qualRow);
if (res->status != WALRCV_OK_TUPLES)
ereport(ERROR,
@@ -993,21 +1010,31 @@ fetch_remote_table_info(char *nspname, char *relname,
slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple);
while (tuplestore_gettupleslot(res->tuplestore, true, false, slot))
{
+ PublicationInfo *pubinfo;
Datum rf = slot_getattr(slot, 1, &isnull);
+ Datum cl;
+
+ pubinfo = (PublicationInfo *) palloc0(sizeof(PublicationInfo));
+
+ if (!isnull)
+ pubinfo->rowfilter = (Node *) makeString(TextDatumGetCString(rf));
+
+ cl = slot_getattr(slot, 2, &isnull);
if (!isnull)
- *qual = lappend(*qual, makeString(TextDatumGetCString(rf)));
- else
{
- /* Ignore filters and cleanup as necessary. */
- if (*qual)
+ int i;
+ int2vector *prattrs = (int2vector *) cl;
+
+ for (i = 0; i < prattrs->dim1; i++)
{
- list_free_deep(*qual);
- *qual = NIL;
+ pubinfo->columns = bms_add_member(pubinfo->columns,
+ prattrs->values[i]);
}
- break;
}
+ *pubinfos = lappend(*pubinfos, pubinfo);
+
ExecClearTuple(slot);
}
ExecDropSingleTupleTableSlot(slot);
@@ -1028,7 +1055,7 @@ copy_table(Relation rel)
{
LogicalRepRelMapEntry *relmapentry;
LogicalRepRelation lrel;
- List *qual = NIL;
+ List *pubinfos = NIL;
WalRcvExecResult *res;
StringInfoData cmd;
CopyFromState cstate;
@@ -1037,7 +1064,7 @@ copy_table(Relation rel)
/* Get the publisher relation info. */
fetch_remote_table_info(get_namespace_name(RelationGetNamespace(rel)),
- RelationGetRelationName(rel), &lrel, &qual);
+ RelationGetRelationName(rel), &lrel, &pubinfos);
/* Put the relation into relmap. */
logicalrep_relmap_update(&lrel);
@@ -1050,7 +1077,9 @@ copy_table(Relation rel)
initStringInfo(&cmd);
/* Regular table with no row filter */
- if (lrel.relkind == RELKIND_RELATION && qual == NIL)
+ /* FIXME pubinfos is never NULL now, need to detect absence of row filters
+ * in a different way */
+ if (lrel.relkind == RELKIND_RELATION && pubinfos == NIL)
{
appendStringInfo(&cmd, "COPY %s (",
quote_qualified_identifier(lrel.nspname, lrel.relname));
@@ -1076,11 +1105,55 @@ copy_table(Relation rel)
* (SELECT ...), but we can't just do SELECT * because we need to not
* copy generated columns. For tables with any row filters, build a
* SELECT query with OR'ed row filters for COPY.
+ *
+ * FIXME can be simplified if all subscriptions have the same column
+ * list (or no column list), in which case we don't need the CASE
+ * expressions at all.
*/
appendStringInfoString(&cmd, "COPY (SELECT ");
for (int i = 0; i < lrel.natts; i++)
{
- appendStringInfoString(&cmd, quote_identifier(lrel.attnames[i]));
+ ListCell *lc;
+ StringInfoData qual;
+ bool no_filter = false;
+ bool is_first = true;
+
+ initStringInfo(&qual);
+
+ /* find all row filters for the column, combine them using OR */
+ foreach (lc, pubinfos)
+ {
+ PublicationInfo *pubinfo = (PublicationInfo *) lfirst(lc);
+
+ /* not included in this publication column list */
+ if (pubinfo->columns != NULL &&
+ !bms_is_member(lrel.attnums[i], pubinfo->columns))
+ continue;
+
+ /* covered by this publication, is there an expression? */
+ if (pubinfo->rowfilter == NULL)
+ {
+ no_filter = true;
+ break;
+ }
+
+ if (is_first)
+ {
+ appendStringInfo(&qual, "%s", strVal(pubinfo->rowfilter));
+ is_first = false;
+ }
+ else
+ appendStringInfo(&qual, " OR %s", strVal(pubinfo->rowfilter));
+ }
+
+ if (no_filter)
+ appendStringInfoString(&cmd, quote_identifier(lrel.attnames[i]));
+ else
+ appendStringInfo(&cmd, "(CASE WHEN (%s) THEN %s ELSE NULL END) AS %s",
+ qual.data,
+ quote_identifier(lrel.attnames[i]),
+ quote_identifier(lrel.attnames[i]));
+
if (i < lrel.natts - 1)
appendStringInfoString(&cmd, ", ");
}
@@ -1095,6 +1168,24 @@ copy_table(Relation rel)
appendStringInfoString(&cmd, "ONLY ");
appendStringInfoString(&cmd, quote_qualified_identifier(lrel.nspname, lrel.relname));
+
+ {
+ List *qual = NIL;
+ ListCell *lc;
+
+ foreach (lc, pubinfos)
+ {
+ PublicationInfo *pubinfo = (PublicationInfo *) lfirst(lc);
+
+ if (pubinfo->rowfilter == NULL)
+ {
+ qual = NIL;
+ break;
+ }
+
+ qual = lappend(qual, pubinfo->rowfilter);
+ }
+
/* list of OR'ed filters */
if (qual != NIL)
{
@@ -1110,6 +1201,8 @@ copy_table(Relation rel)
list_free_deep(qual);
}
+ }
+
appendStringInfoString(&cmd, ") TO STDOUT");
}
res = walrcv_exec(LogRepWorkerWalRcvConn, cmd.data, 0, NULL);
diff --git a/src/backend/replication/pgoutput/pgoutput.c b/src/backend/replication/pgoutput/pgoutput.c
index b197bfd565d..85456e6d9f5 100644
--- a/src/backend/replication/pgoutput/pgoutput.c
+++ b/src/backend/replication/pgoutput/pgoutput.c
@@ -137,13 +137,13 @@ typedef struct RelationSyncEntry
PublicationActions pubactions;
/*
- * ExprState array for row filter. Different publication actions don't
- * allow multiple expressions to always be combined into one, because
- * updates or deletes restrict the column in expression to be part of the
- * replica identity index whereas inserts do not have this restriction, so
- * there is one ExprState per publication action.
+ * Info about row filters and column lists for each publication this
+ * relation is included in. We keep a list with per-publication info in
+ * order to calculate appropriate column list depending on which row
+ * filter(s) match. Each element contains an ExprState for the filter
+ * and column list.
*/
- ExprState *exprstate[NUM_ROWFILTER_PUBACTIONS];
+ dlist_head pubinfos; /* one per publication */
EState *estate; /* executor state used for row filter */
TupleTableSlot *new_slot; /* slot for storing new tuple */
TupleTableSlot *old_slot; /* slot for storing old tuple */
@@ -208,6 +208,29 @@ typedef struct PGOutputTxnData
* been sent */
} PGOutputTxnData;
+/*
+ * Info use to track and evaluate row filters for each publication the relation
+ * is included in, and calculat ethe column list.
+ */
+typedef struct PublicationInfo {
+
+ /* doubly-linked list */
+ dlist_node node;
+
+ /* publication OID (XXX not really needed) */
+ Oid oid;
+
+ /* row filter (expression state) */
+ ExprState *rowfilter;
+
+ /* column list */
+ Bitmapset *columns;
+
+ /* actions published by the publication */
+ PublicationActions pubactions;
+
+} PublicationInfo;
+
/* Map used to remember which relation schemas we sent. */
static HTAB *RelationSyncCache = NULL;
@@ -235,12 +258,8 @@ static bool pgoutput_row_filter_exec_expr(ExprState *state,
static bool pgoutput_row_filter(Relation relation, TupleTableSlot *old_slot,
TupleTableSlot **new_slot_ptr,
RelationSyncEntry *entry,
- ReorderBufferChangeType *action);
-
-/* column list routines */
-static void pgoutput_column_list_init(PGOutputData *data,
- List *publications,
- RelationSyncEntry *entry);
+ ReorderBufferChangeType *action,
+ Bitmapset **column_list);
/*
* Specify output plugin callbacks
@@ -822,18 +841,24 @@ pgoutput_ensure_entry_cxt(PGOutputData *data, RelationSyncEntry *entry)
}
/*
- * Initialize the row filter.
+ * Initialize the row filter and column list.
+ *
+ * Prepare information (ExprState, etc) used to evaluate per-publication row
+ * filters, and column lists.
+ *
+ * We also calculate a "total" column list as a union of all per-publication
+ * column lists, irrespectedly of row filters. This is used to send schema
+ * for the relsync entry, etc.
*/
static void
pgoutput_row_filter_init(PGOutputData *data, List *publications,
RelationSyncEntry *entry)
{
ListCell *lc;
- List *rfnodes[] = {NIL, NIL, NIL}; /* One per pubaction */
- bool no_filter[] = {false, false, false}; /* One per pubaction */
MemoryContext oldctx;
- int idx;
- bool has_filter = true;
+ bool all_columns = false;
+
+ dlist_init(&entry->pubinfos);
/*
* Find if there are any row filters for this relation. If there are, then
@@ -855,7 +880,12 @@ pgoutput_row_filter_init(PGOutputData *data, List *publications,
Publication *pub = lfirst(lc);
HeapTuple rftuple = NULL;
Datum rfdatum = 0;
+ Datum cfdatum = 0;
bool pub_no_filter = false;
+ bool pub_no_list = false;
+ Relation relation = RelationIdGetRelation(entry->publish_as_relid);
+
+ PublicationInfo *pubinfo = NULL;
if (pub->alltables)
{
@@ -865,6 +895,7 @@ pgoutput_row_filter_init(PGOutputData *data, List *publications,
* publications it does).
*/
pub_no_filter = true;
+ pub_no_list = true;
}
else
{
@@ -881,191 +912,75 @@ pgoutput_row_filter_init(PGOutputData *data, List *publications,
rfdatum = SysCacheGetAttr(PUBLICATIONRELMAP, rftuple,
Anum_pg_publication_rel_prqual,
&pub_no_filter);
+
+ /*
+ * Lookup the column list attribute.
+ *
+ * Null indicates no list.
+ */
+ cfdatum = SysCacheGetAttr(PUBLICATIONRELMAP, rftuple,
+ Anum_pg_publication_rel_prattrs,
+ &pub_no_list);
}
else
{
pub_no_filter = true;
+ pub_no_list = true;
}
}
- if (pub_no_filter)
- {
- if (rftuple)
- ReleaseSysCache(rftuple);
-
- no_filter[PUBACTION_INSERT] |= pub->pubactions.pubinsert;
- no_filter[PUBACTION_UPDATE] |= pub->pubactions.pubupdate;
- no_filter[PUBACTION_DELETE] |= pub->pubactions.pubdelete;
-
- /*
- * Quick exit if all the DML actions are publicized via this
- * publication.
- */
- if (no_filter[PUBACTION_INSERT] &&
- no_filter[PUBACTION_UPDATE] &&
- no_filter[PUBACTION_DELETE])
- {
- has_filter = false;
- break;
- }
-
- /* No additional work for this publication. Next one. */
- continue;
- }
-
- /* Form the per pubaction row filter lists. */
- if (pub->pubactions.pubinsert && !no_filter[PUBACTION_INSERT])
- rfnodes[PUBACTION_INSERT] = lappend(rfnodes[PUBACTION_INSERT],
- TextDatumGetCString(rfdatum));
- if (pub->pubactions.pubupdate && !no_filter[PUBACTION_UPDATE])
- rfnodes[PUBACTION_UPDATE] = lappend(rfnodes[PUBACTION_UPDATE],
- TextDatumGetCString(rfdatum));
- if (pub->pubactions.pubdelete && !no_filter[PUBACTION_DELETE])
- rfnodes[PUBACTION_DELETE] = lappend(rfnodes[PUBACTION_DELETE],
- TextDatumGetCString(rfdatum));
-
- ReleaseSysCache(rftuple);
- } /* loop all subscribed publications */
-
- /* Clean the row filter */
- for (idx = 0; idx < NUM_ROWFILTER_PUBACTIONS; idx++)
- {
- if (no_filter[idx])
- {
- list_free_deep(rfnodes[idx]);
- rfnodes[idx] = NIL;
- }
- }
-
- if (has_filter)
- {
- Relation relation = RelationIdGetRelation(entry->publish_as_relid);
-
pgoutput_ensure_entry_cxt(data, entry);
- /*
- * Now all the filters for all pubactions are known. Combine them when
- * their pubactions are the same.
- */
oldctx = MemoryContextSwitchTo(entry->entry_cxt);
- entry->estate = create_estate_for_relation(relation);
- for (idx = 0; idx < NUM_ROWFILTER_PUBACTIONS; idx++)
- {
- List *filters = NIL;
- Expr *rfnode;
- if (rfnodes[idx] == NIL)
- continue;
-
- foreach(lc, rfnodes[idx])
- filters = lappend(filters, stringToNode((char *) lfirst(lc)));
-
- /* combine the row filter and cache the ExprState */
- rfnode = make_orclause(filters);
- entry->exprstate[idx] = ExecPrepareExpr(rfnode, entry->estate);
- } /* for each pubaction */
- MemoryContextSwitchTo(oldctx);
-
- RelationClose(relation);
- }
-}
+ pubinfo = (PublicationInfo *) palloc0(sizeof(PublicationInfo));
-/*
- * Initialize the column list.
- */
-static void
-pgoutput_column_list_init(PGOutputData *data, List *publications,
- RelationSyncEntry *entry)
-{
- ListCell *lc;
+ pubinfo->oid = pub->oid;
+ pubinfo->pubactions = pub->pubactions;
- /*
- * Find if there are any column lists for this relation. If there are,
- * build a bitmap merging all the column lists.
- *
- * All the given publication-table mappings must be checked.
- *
- * Multiple publications might have multiple column lists for this relation.
- *
- * FOR ALL TABLES and FOR ALL TABLES IN SCHEMA implies "don't use column
- * list" so it takes precedence.
- */
- foreach(lc, publications)
- {
- Publication *pub = lfirst(lc);
- HeapTuple cftuple = NULL;
- Datum cfdatum = 0;
+ if (!pub_no_filter)
+ {
+ entry->estate = create_estate_for_relation(relation);
- /*
- * Assume there's no column list. Only if we find pg_publication_rel
- * entry with a column list we'll switch it to false.
- */
- bool pub_no_list = true;
+ pubinfo->rowfilter
+ = ExecPrepareExpr(stringToNode(TextDatumGetCString(rfdatum)),
+ entry->estate);
+ }
/*
- * If the publication is FOR ALL TABLES then it is treated the same as if
- * there are no column lists (even if other publications have a list).
+ * Build the column list bitmap in the per-entry context.
+ *
+ * We need to merge column lists from all publications, so we
+ * update the same bitmapset. If the column list is null, we
+ * interpret it as replicating all columns.
*/
- if (!pub->alltables)
+ pubinfo->columns = NULL;
+ if (!pub_no_list) /* when not null */
{
- /*
- * Check for the presence of a column list in this publication.
- *
- * Note: If we find no pg_publication_rel row, it's a publication
- * defined for a whole schema, so it can't have a column list, just
- * like a FOR ALL TABLES publication.
- */
- cftuple = SearchSysCache2(PUBLICATIONRELMAP,
- ObjectIdGetDatum(entry->publish_as_relid),
- ObjectIdGetDatum(pub->oid));
-
- if (HeapTupleIsValid(cftuple))
- {
- /*
- * Lookup the column list attribute.
- *
- * Note: We update the pub_no_list value directly, because if
- * the value is NULL, we have no list (and vice versa).
- */
- cfdatum = SysCacheGetAttr(PUBLICATIONRELMAP, cftuple,
- Anum_pg_publication_rel_prattrs,
- &pub_no_list);
-
- /*
- * Build the column list bitmap in the per-entry context.
- *
- * We need to merge column lists from all publications, so we
- * update the same bitmapset. If the column list is null, we
- * interpret it as replicating all columns.
- */
- if (!pub_no_list) /* when not null */
- {
- pgoutput_ensure_entry_cxt(data, entry);
+ pubinfo->columns = pub_collist_to_bitmapset(pubinfo->columns,
+ cfdatum,
+ entry->entry_cxt);
- entry->columns = pub_collist_to_bitmapset(entry->columns,
- cfdatum,
- entry->entry_cxt);
- }
- }
+ entry->columns = pub_collist_to_bitmapset(entry->columns,
+ cfdatum,
+ entry->entry_cxt);
}
+ else
+ all_columns = true;
- /*
- * Found a publication with no column list, so we're done. But first
- * discard column list we might have from preceding publications.
- */
- if (pub_no_list)
- {
- if (cftuple)
- ReleaseSysCache(cftuple);
+ if (HeapTupleIsValid(rftuple))
+ ReleaseSysCache(rftuple);
- bms_free(entry->columns);
- entry->columns = NULL;
+ MemoryContextSwitchTo(oldctx);
+ RelationClose(relation);
- break;
- }
+ dlist_push_tail(&entry->pubinfos, &pubinfo->node);
- ReleaseSysCache(cftuple);
} /* loop all subscribed publications */
+
+ /* any of the publications replicates all columns */
+ if (all_columns)
+ entry->columns = NULL;
}
/*
@@ -1115,7 +1030,8 @@ init_tuple_slot(PGOutputData *data, Relation relation,
}
/*
- * Change is checked against the row filter if any.
+ * Change is checked against the row filter if any, and calculate the column
+ * list applicable to the operation (with respect to matching row filters).
*
* Returns true if the change is to be replicated, else false.
*
@@ -1136,6 +1052,8 @@ init_tuple_slot(PGOutputData *data, Relation relation,
*
* The new action is updated in the action parameter.
*
+ * The calculated column list is returned in the column_list parameter.
+ *
* The new slot could be updated when transforming the UPDATE into INSERT,
* because the original new tuple might not have column values from the replica
* identity.
@@ -1167,17 +1085,21 @@ init_tuple_slot(PGOutputData *data, Relation relation,
static bool
pgoutput_row_filter(Relation relation, TupleTableSlot *old_slot,
TupleTableSlot **new_slot_ptr, RelationSyncEntry *entry,
- ReorderBufferChangeType *action)
+ ReorderBufferChangeType *action, Bitmapset **column_list)
{
TupleDesc desc;
int i;
- bool old_matched,
- new_matched,
+ bool old_matched_any = false,
+ new_matched_any = false,
result;
- TupleTableSlot *tmp_new_slot;
+ TupleTableSlot *tmp_new_slot = NULL;
TupleTableSlot *new_slot = *new_slot_ptr;
- ExprContext *ecxt;
- ExprState *filter_exprstate;
+ dlist_iter iter;
+ bool matching = false;
+
+ /* Column list calculated from publications matching the row filter. */
+ Bitmapset *columns = NULL;
+ bool all_columns = false;
/*
* We need this map to avoid relying on ReorderBufferChangeType enums
@@ -1195,115 +1117,191 @@ pgoutput_row_filter(Relation relation, TupleTableSlot *old_slot,
Assert(new_slot || old_slot);
- /* Get the corresponding row filter */
- filter_exprstate = entry->exprstate[map_changetype_pubaction[*action]];
+ dlist_foreach(iter, &entry->pubinfos)
+ {
+ ExprContext *ecxt;
+ bool old_matched,
+ new_matched;
- /* Bail out if there is no row filter */
- if (!filter_exprstate)
- return true;
+ PublicationInfo *pubinfo
+ = dlist_container(PublicationInfo, node, iter.cur);
- elog(DEBUG3, "table \"%s.%s\" has row filter",
- get_namespace_name(RelationGetNamespace(relation)),
- RelationGetRelationName(relation));
+ /* ignore publications not replicating this action */
+ if ((*action == REORDER_BUFFER_CHANGE_INSERT) &&
+ (!pubinfo->pubactions.pubinsert))
+ continue;
+ else if ((*action == REORDER_BUFFER_CHANGE_UPDATE) &&
+ (!pubinfo->pubactions.pubupdate))
+ continue;
+ else if ((*action == REORDER_BUFFER_CHANGE_DELETE) &&
+ (!pubinfo->pubactions.pubdelete))
+ continue;
- ResetPerTupleExprContext(entry->estate);
+ if (!pubinfo->rowfilter)
+ {
+ matching = true;
- ecxt = GetPerTupleExprContext(entry->estate);
+ /*
+ * Update/merge the column list.
+ *
+ * If the publication has no column list, we interpret it as a list
+ * with all columns. Otherwise we just add it to the bitmap.
+ *
+ * FIXME This is repeated in three places. Maybe refactor?
+ */
+ if (!pubinfo->columns)
+ {
+ all_columns = true;
+ bms_free(columns);
+ columns = NULL;
+ }
+ else if (!all_columns)
+ columns = bms_union(columns, pubinfo->columns);
- /*
- * For the following occasions where there is only one tuple, we can
- * evaluate the row filter for that tuple and return.
- *
- * For inserts, we only have the new tuple.
- *
- * For updates, we can have only a new tuple when none of the replica
- * identity columns changed and none of those columns have external data
- * but we still need to evaluate the row filter for the new tuple as the
- * existing values of those columns might not match the filter. Also, users
- * can use constant expressions in the row filter, so we anyway need to
- * evaluate it for the new tuple.
- *
- * For deletes, we only have the old tuple.
- */
- if (!new_slot || !old_slot)
- {
- ecxt->ecxt_scantuple = new_slot ? new_slot : old_slot;
- result = pgoutput_row_filter_exec_expr(filter_exprstate, ecxt);
+ continue;
+ }
- return result;
- }
+ elog(DEBUG3, "table \"%s.%s\" has row filter",
+ get_namespace_name(RelationGetNamespace(relation)),
+ RelationGetRelationName(relation));
- /*
- * Both the old and new tuples must be valid only for updates and need to
- * be checked against the row filter.
- */
- Assert(map_changetype_pubaction[*action] == PUBACTION_UPDATE);
+ ResetPerTupleExprContext(entry->estate);
+
+ ecxt = GetPerTupleExprContext(entry->estate);
- slot_getallattrs(new_slot);
- slot_getallattrs(old_slot);
+ /*
+ * For the following occasions where there is only one tuple, we can
+ * evaluate the row filter for that tuple and return.
+ *
+ * For inserts, we only have the new tuple.
+ *
+ * For updates, we can have only a new tuple when none of the replica
+ * identity columns changed and none of those columns have external data
+ * but we still need to evaluate the row filter for the new tuple as the
+ * existing values of those columns might not match the filter. Also, users
+ * can use constant expressions in the row filter, so we anyway need to
+ * evaluate it for the new tuple.
+ *
+ * For deletes, we only have the old tuple.
+ */
+ if (!new_slot || !old_slot)
+ {
+ ecxt->ecxt_scantuple = new_slot ? new_slot : old_slot;
+ result = pgoutput_row_filter_exec_expr(pubinfo->rowfilter, ecxt);
- tmp_new_slot = NULL;
- desc = RelationGetDescr(relation);
+ matching |= result;
- /*
- * The new tuple might not have all the replica identity columns, in which
- * case it needs to be copied over from the old tuple.
- */
- for (i = 0; i < desc->natts; i++)
- {
- Form_pg_attribute att = TupleDescAttr(desc, i);
+ /*
+ * FIXME refactor to reuse this code in multiple places
+ *
+ * XXX Should this only update column list when using new slot?
+ * If evaluationg old slot, that's delete, no?
+ */
+ if (result)
+ {
+ if (!pubinfo->columns)
+ {
+ all_columns = true;
+ bms_free(columns);
+ columns = NULL;
+ }
+ else if (!all_columns)
+ {
+ columns = bms_union(columns, pubinfo->columns);
+ }
+ }
+
+ continue;
+ }
/*
- * if the column in the new tuple or old tuple is null, nothing to do
+ * Both the old and new tuples must be valid only for updates and need to
+ * be checked against the row filter.
*/
- if (new_slot->tts_isnull[i] || old_slot->tts_isnull[i])
- continue;
+ Assert(map_changetype_pubaction[*action] == PUBACTION_UPDATE);
+
+ slot_getallattrs(new_slot);
+ slot_getallattrs(old_slot);
+
+ tmp_new_slot = NULL;
+ desc = RelationGetDescr(relation);
/*
- * Unchanged toasted replica identity columns are only logged in the
- * old tuple. Copy this over to the new tuple. The changed (or WAL
- * Logged) toast values are always assembled in memory and set as
- * VARTAG_INDIRECT. See ReorderBufferToastReplace.
+ * The new tuple might not have all the replica identity columns, in which
+ * case it needs to be copied over from the old tuple.
*/
- if (att->attlen == -1 &&
- VARATT_IS_EXTERNAL_ONDISK(new_slot->tts_values[i]) &&
- !VARATT_IS_EXTERNAL_ONDISK(old_slot->tts_values[i]))
+ for (i = 0; i < desc->natts; i++)
{
- if (!tmp_new_slot)
+ Form_pg_attribute att = TupleDescAttr(desc, i);
+
+ /*
+ * if the column in the new tuple or old tuple is null, nothing to do
+ */
+ if (new_slot->tts_isnull[i] || old_slot->tts_isnull[i])
+ continue;
+
+ /*
+ * Unchanged toasted replica identity columns are only logged in the
+ * old tuple. Copy this over to the new tuple. The changed (or WAL
+ * Logged) toast values are always assembled in memory and set as
+ * VARTAG_INDIRECT. See ReorderBufferToastReplace.
+ */
+ if (att->attlen == -1 &&
+ VARATT_IS_EXTERNAL_ONDISK(new_slot->tts_values[i]) &&
+ !VARATT_IS_EXTERNAL_ONDISK(old_slot->tts_values[i]))
{
- tmp_new_slot = MakeSingleTupleTableSlot(desc, &TTSOpsVirtual);
- ExecClearTuple(tmp_new_slot);
+ if (!tmp_new_slot)
+ {
+ tmp_new_slot = MakeSingleTupleTableSlot(desc, &TTSOpsVirtual);
+ ExecClearTuple(tmp_new_slot);
- memcpy(tmp_new_slot->tts_values, new_slot->tts_values,
- desc->natts * sizeof(Datum));
- memcpy(tmp_new_slot->tts_isnull, new_slot->tts_isnull,
- desc->natts * sizeof(bool));
- }
+ memcpy(tmp_new_slot->tts_values, new_slot->tts_values,
+ desc->natts * sizeof(Datum));
+ memcpy(tmp_new_slot->tts_isnull, new_slot->tts_isnull,
+ desc->natts * sizeof(bool));
+ }
- tmp_new_slot->tts_values[i] = old_slot->tts_values[i];
- tmp_new_slot->tts_isnull[i] = old_slot->tts_isnull[i];
+ tmp_new_slot->tts_values[i] = old_slot->tts_values[i];
+ tmp_new_slot->tts_isnull[i] = old_slot->tts_isnull[i];
+ }
}
- }
- ecxt->ecxt_scantuple = old_slot;
- old_matched = pgoutput_row_filter_exec_expr(filter_exprstate, ecxt);
+ ecxt->ecxt_scantuple = old_slot;
- if (tmp_new_slot)
- {
- ExecStoreVirtualTuple(tmp_new_slot);
- ecxt->ecxt_scantuple = tmp_new_slot;
- }
- else
- ecxt->ecxt_scantuple = new_slot;
+ old_matched = pgoutput_row_filter_exec_expr(pubinfo->rowfilter, ecxt);
+ old_matched_any |= old_matched;
+
+ if (tmp_new_slot)
+ {
+ ExecStoreVirtualTuple(tmp_new_slot);
+ ecxt->ecxt_scantuple = tmp_new_slot;
+ }
+ else
+ ecxt->ecxt_scantuple = new_slot;
- new_matched = pgoutput_row_filter_exec_expr(filter_exprstate, ecxt);
+ new_matched = pgoutput_row_filter_exec_expr(pubinfo->rowfilter, ecxt);
+ new_matched_any |= new_matched;
- /*
- * Case 1: if both tuples don't match the row filter, bailout. Send
- * nothing.
- */
- if (!old_matched && !new_matched)
- return false;
+ /*
+ * Case 1: if both tuples don't match the row filter, bailout. Send
+ * nothing.
+ */
+ if (!old_matched && !new_matched)
+ continue; /* continue with the next row filter */
+
+ /*
+ * Case 4: if both tuples match the row filter, transformation isn't
+ * required. (*action is default UPDATE).
+ */
+ if (!pubinfo->columns)
+ {
+ all_columns = true;
+ bms_free(columns);
+ columns = NULL;
+ }
+ else if (!all_columns && new_matched)
+ columns = bms_union(columns, pubinfo->columns);
+ }
/*
* Case 2: if the old tuple doesn't satisfy the row filter but the new
@@ -1314,9 +1312,10 @@ pgoutput_row_filter(Relation relation, TupleTableSlot *old_slot,
* while inserting the tuple in the downstream node, we have all the
* required column values.
*/
- if (!old_matched && new_matched)
+ if (!old_matched_any && new_matched_any)
{
*action = REORDER_BUFFER_CHANGE_INSERT;
+ matching = true;
if (tmp_new_slot)
*new_slot_ptr = tmp_new_slot;
@@ -1329,15 +1328,18 @@ pgoutput_row_filter(Relation relation, TupleTableSlot *old_slot,
* This transformation does not require another tuple. The Old tuple will
* be used for DELETE.
*/
- else if (old_matched && !new_matched)
+ else if (old_matched_any && !new_matched_any)
+ {
*action = REORDER_BUFFER_CHANGE_DELETE;
+ matching = true;
+ }
+ else if (old_matched_any && new_matched_any)
+ matching = true;
- /*
- * Case 4: if both tuples match the row filter, transformation isn't
- * required. (*action is default UPDATE).
- */
+ if (column_list)
+ *column_list = columns;
- return true;
+ return matching;
}
/*
@@ -1359,6 +1361,7 @@ pgoutput_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn,
ReorderBufferChangeType action = change->action;
TupleTableSlot *old_slot = NULL;
TupleTableSlot *new_slot = NULL;
+ Bitmapset *columns = NULL;
if (!is_publishable_relation(relation))
return;
@@ -1423,7 +1426,7 @@ pgoutput_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn,
/* Check row filter */
if (!pgoutput_row_filter(targetrel, NULL, &new_slot, relentry,
- &action))
+ &action, &columns))
break;
/*
@@ -1444,7 +1447,8 @@ pgoutput_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn,
OutputPluginPrepareWrite(ctx, true);
logicalrep_write_insert(ctx->out, xid, targetrel, new_slot,
- data->binary, relentry->columns);
+ data->binary,
+ relentry->columns, columns);
OutputPluginWrite(ctx, true);
break;
case REORDER_BUFFER_CHANGE_UPDATE:
@@ -1483,7 +1487,7 @@ pgoutput_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn,
/* Check row filter */
if (!pgoutput_row_filter(targetrel, old_slot, &new_slot,
- relentry, &action))
+ relentry, &action, &columns))
break;
/* Send BEGIN if we haven't yet */
@@ -1503,12 +1507,12 @@ pgoutput_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn,
case REORDER_BUFFER_CHANGE_INSERT:
logicalrep_write_insert(ctx->out, xid, targetrel,
new_slot, data->binary,
- relentry->columns);
+ relentry->columns, columns);
break;
case REORDER_BUFFER_CHANGE_UPDATE:
logicalrep_write_update(ctx->out, xid, targetrel,
old_slot, new_slot, data->binary,
- relentry->columns);
+ relentry->columns, columns);
break;
case REORDER_BUFFER_CHANGE_DELETE:
logicalrep_write_delete(ctx->out, xid, targetrel,
@@ -1547,7 +1551,7 @@ pgoutput_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn,
/* Check row filter */
if (!pgoutput_row_filter(targetrel, old_slot, &new_slot,
- relentry, &action))
+ relentry, &action, NULL))
break;
/* Send BEGIN if we haven't yet */
@@ -1977,7 +1981,7 @@ get_rel_sync_entry(PGOutputData *data, Relation relation)
entry->pubactions.pubdelete = entry->pubactions.pubtruncate = false;
entry->new_slot = NULL;
entry->old_slot = NULL;
- memset(entry->exprstate, 0, sizeof(entry->exprstate));
+ dlist_init(&entry->pubinfos);
entry->entry_cxt = NULL;
entry->publish_as_relid = InvalidOid;
entry->columns = NULL;
@@ -2056,7 +2060,7 @@ get_rel_sync_entry(PGOutputData *data, Relation relation)
entry->entry_cxt = NULL;
entry->estate = NULL;
- memset(entry->exprstate, 0, sizeof(entry->exprstate));
+ dlist_init(&entry->pubinfos);
/*
* Build publication cache. We can't use one provided by relcache as
@@ -2192,11 +2196,8 @@ get_rel_sync_entry(PGOutputData *data, Relation relation)
/* Initialize the tuple slot and map */
init_tuple_slot(data, relation, entry);
- /* Initialize the row filter */
+ /* Initialize the row filter and column list info */
pgoutput_row_filter_init(data, rel_publications, entry);
-
- /* Initialize the column list */
- pgoutput_column_list_init(data, rel_publications, entry);
}
list_free(pubids);
diff --git a/src/include/replication/logicalproto.h b/src/include/replication/logicalproto.h
index a771ab8ff33..1a3f4f34bf4 100644
--- a/src/include/replication/logicalproto.h
+++ b/src/include/replication/logicalproto.h
@@ -104,6 +104,7 @@ typedef struct LogicalRepRelation
char *relname; /* relation name */
int natts; /* number of columns */
char **attnames; /* column names */
+ int16 *attnums; /* column attnums */
Oid *atttyps; /* column types */
char replident; /* replica identity */
char relkind; /* remote relation kind */
@@ -209,12 +210,14 @@ extern char *logicalrep_read_origin(StringInfo in, XLogRecPtr *origin_lsn);
extern void logicalrep_write_insert(StringInfo out, TransactionId xid,
Relation rel,
TupleTableSlot *newslot,
- bool binary, Bitmapset *columns);
+ bool binary, Bitmapset *schema_columns,
+ Bitmapset *columns);
extern LogicalRepRelId logicalrep_read_insert(StringInfo in, LogicalRepTupleData *newtup);
extern void logicalrep_write_update(StringInfo out, TransactionId xid,
Relation rel,
TupleTableSlot *oldslot,
- TupleTableSlot *newslot, bool binary, Bitmapset *columns);
+ TupleTableSlot *newslot, bool binary,
+ Bitmapset *schema_columns, Bitmapset *columns);
extern LogicalRepRelId logicalrep_read_update(StringInfo in,
bool *has_oldtuple, LogicalRepTupleData *oldtup,
LogicalRepTupleData *newtup);
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-02 09:35 Tomas Vondra <[email protected]>
0 siblings, 2 replies; 58+ messages in thread
From: Tomas Vondra @ 2022-05-02 09:35 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On 5/2/22 07:31, Amit Kapila wrote:
> On Mon, May 2, 2022 at 3:27 AM Tomas Vondra
> <[email protected]> wrote:
>>
>> On 4/30/22 12:11, Amit Kapila wrote:
>>> On Sat, Apr 30, 2022 at 3:01 PM Alvaro Herrera <[email protected]> wrote:
>>>>
>>>> My proposal is that if users want to define multiple publications, and
>>>> their definitions conflict in a way that would behave ridiculously (==
>>>> bound to cause data inconsistencies eventually), an error should be
>>>> thrown. Maybe we will not be able to catch all bogus cases, but we can
>>>> be prepared for the most obvious ones, and patch later when we find
>>>> others.
>>>>
>>>
>>> I agree with throwing errors for obvious/known bogus cases but do we
>>> want to throw errors or restrict the combining of column lists when
>>> row filters are present in all cases? See some examples [1 ] where it
>>> may be valid to combine them.
>>>
>>
>> I think there are three challenges:
>>
>> (a) Deciding what's an obvious bug or an unsupported case (e.g. because
>> it's not clear what's the correct behavior / way to merge column lists).
>>
>> (b) When / where to detect the issue.
>>
>> (c) Making sure this does not break/prevent existing use cases.
>>
>>
>> As I said before [1], I think the issue stems from essentially allowing
>> DML to have different row filters / column lists. So we could forbid
>> publications to specify WITH (publish=...) and one of the two features,
>>
>
> I don't think this is feasible for row filters because that would mean
> publishing all actions because we have a restriction that all columns
> referenced in the row filter expression are part of the REPLICA
> IDENTITY index. This restriction is only valid for updates/deletes, so
> if we allow all pubactions then this will be imposed on inserts as
> well. A similar restriction is there for column lists as well, so I
> don't think we can do it there as well. Do you have some idea to
> address it?
>
No, I haven't thought about how exactly to implement this, and I have
not thought about how to deal with the replica identity issues. My
thoughts were that we'd only really need this for tables with row
filters and/or column lists, treating it as a cost of those features.
But yeah, it seems annoying.
>> or make sure subscription does not combine multiple such publications.
>>
>
> Yeah, or don't allow to define such publications in the first place so
> that different subscriptions can't combine them but I guess that might
> forbid some useful cases as well where publication may not get
> combined with other publications.
>
But how would you check that? You don't know which publications will be
combined by a subscription until you create the subscription, right?
>> The second option has the annoying consequence that it makes this
>> useless for the "data redaction" use case I described in [2], because
>> that relies on combining multiple publications.
>>
>
> True, but as a workaround users can create different subscriptions for
> different publications.
>
Won't that replicate duplicate data, when the row filters re not
mutually exclusive?
>> Furthermore, what if the publications change after the subscriptions get
>> created? Will we be able to detect the error etc.?
>>
>
> I think from that apart from 'Create Subscription', the same check
> needs to be added for Alter Subscription ... Refresh, Alter
> Subscription ... Enable.
>
> In the publication side, we need an additional check in Alter
> Publication ... SET table variant. One idea is that we get all other
> publications for which the corresponding relation is defined. And then
> if we find anything which we don't want to allow then we can throw an
> error. This will forbid some useful cases as well as mentioned above.
> So, the other possibility is to expose all publications for a
> walsender, and then we can find the exact set of publications where
> the current publication is used with other publications and we can
> check only those publications. So, if we have three walsenders
> (walsnd1: pub1, pub2; walsnd2 pub2; walsnd3: pub2, pub3) in the system
> and we are currently altering publication pub1 then we need to check
> only pub3 for any conflicting conditions. Yet another simple way could
> be that we don't allow to change column list via Alter Publication ...
> Set variant because the other variants anyway need REFRESH publication
> which we have covered.
>
> I think it is tricky to decide what exactly we want to forbid, so, we
> may want to follow something simple like if the column list and row
> filters for a table are different in the required set of publications
> then we treat it as an unsupported case. I think this will prohibit
> some useful cases but should probably forbid the cases we are worried
> about here.
>
I don't have a clear idea on what the right tradeoff is :-(
Maybe we're digressing a bit from the stuff Alvaro complained about
initially. Arguably the existing column list behavior is surprising and
would not work with reasonable use cases. So let's fix it.
But maybe you're right validating row filters is a step too far. Yes,
users may define strange combinations of publications, but is that
really an issue we have to solve?
regards
--
Tomas Vondra
EnterpriseDB: http://www.enterprisedb.com
The Enterprise PostgreSQL Company
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-02 10:17 Alvaro Herrera <[email protected]>
parent: Tomas Vondra <[email protected]>
1 sibling, 1 reply; 58+ messages in thread
From: Alvaro Herrera @ 2022-05-02 10:17 UTC (permalink / raw)
To: Tomas Vondra <[email protected]>; +Cc: Amit Kapila <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On 2022-May-02, Tomas Vondra wrote:
> On 5/2/22 07:31, Amit Kapila wrote:
> > Yeah, or don't allow to define such publications in the first place so
> > that different subscriptions can't combine them but I guess that might
> > forbid some useful cases as well where publication may not get
> > combined with other publications.
>
> But how would you check that? You don't know which publications will be
> combined by a subscription until you create the subscription, right?
... and I think this poses a problem: if the publisher has multiple
publications and the subscriber later uses those to create a combined
subscription, we can check at CREATE SUBSCRIPTION time that they can be
combined correctly. But if the publisher decides to change the
publications changing the rules and they are no longer consistent, can
we throw an error at ALTER PUBLICATION point? If the publisher can
detect that they are being used together by some subscription, then
maybe we can check consistency in the publication side and everything is
all right. But I'm not sure that the publisher knows who is subscribed
to what, so this might not be an option.
The latter ultimately means that we aren't sure that a combined
subscription is safe. And in turn this means that a pg_dump of such a
database cannot be restored (because the CREATE SUBSCRIPTION will be
rejected as being inconsistent).
--
Álvaro Herrera 48°01'N 7°57'E — https://www.EnterpriseDB.com/
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-02 10:23 Tomas Vondra <[email protected]>
parent: Alvaro Herrera <[email protected]>
0 siblings, 2 replies; 58+ messages in thread
From: Tomas Vondra @ 2022-05-02 10:23 UTC (permalink / raw)
To: Alvaro Herrera <[email protected]>; +Cc: Amit Kapila <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On 5/2/22 12:17, Alvaro Herrera wrote:
> On 2022-May-02, Tomas Vondra wrote:
>> On 5/2/22 07:31, Amit Kapila wrote:
>
>>> Yeah, or don't allow to define such publications in the first place so
>>> that different subscriptions can't combine them but I guess that might
>>> forbid some useful cases as well where publication may not get
>>> combined with other publications.
>>
>> But how would you check that? You don't know which publications will be
>> combined by a subscription until you create the subscription, right?
>
> ... and I think this poses a problem: if the publisher has multiple
> publications and the subscriber later uses those to create a combined
> subscription, we can check at CREATE SUBSCRIPTION time that they can be
> combined correctly. But if the publisher decides to change the
> publications changing the rules and they are no longer consistent, can
> we throw an error at ALTER PUBLICATION point? If the publisher can
> detect that they are being used together by some subscription, then
> maybe we can check consistency in the publication side and everything is
> all right. But I'm not sure that the publisher knows who is subscribed
> to what, so this might not be an option.
>
AFAIK we don't track that (publication/subscription mapping). The
publications are listed in publication_names parameter of the
START_REPLICATION command.
> The latter ultimately means that we aren't sure that a combined
> subscription is safe. And in turn this means that a pg_dump of such a
> database cannot be restored (because the CREATE SUBSCRIPTION will be
> rejected as being inconsistent).
>
We could do this check when executing the START_REPLICATION command, no?
regards
--
Tomas Vondra
EnterpriseDB: http://www.enterprisedb.com
The Enterprise PostgreSQL Company
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-02 10:55 Alvaro Herrera <[email protected]>
parent: Tomas Vondra <[email protected]>
1 sibling, 0 replies; 58+ messages in thread
From: Alvaro Herrera @ 2022-05-02 10:55 UTC (permalink / raw)
To: Tomas Vondra <[email protected]>; +Cc: Amit Kapila <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On 2022-May-02, Tomas Vondra wrote:
> On 5/2/22 12:17, Alvaro Herrera wrote:
> > The latter ultimately means that we aren't sure that a combined
> > subscription is safe. And in turn this means that a pg_dump of such a
> > database cannot be restored (because the CREATE SUBSCRIPTION will be
> > rejected as being inconsistent).
>
> We could do this check when executing the START_REPLICATION command, no?
Ah! That sounds like it might work: we throw WARNINGs are CREATE
SUBSCRIPTION (so that users are immediately aware in case something is
going to fail later, but the objects are still created and they can fix
the publications afterwards), but the real ERROR is in START_REPLICATION.
--
Álvaro Herrera PostgreSQL Developer — https://www.EnterpriseDB.com/
"Uno puede defenderse de los ataques; contra los elogios se esta indefenso"
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-02 11:14 Amit Kapila <[email protected]>
parent: Tomas Vondra <[email protected]>
1 sibling, 1 reply; 58+ messages in thread
From: Amit Kapila @ 2022-05-02 11:14 UTC (permalink / raw)
To: Tomas Vondra <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On Mon, May 2, 2022 at 3:53 PM Tomas Vondra
<[email protected]> wrote:
>
> On 5/2/22 12:17, Alvaro Herrera wrote:
> > On 2022-May-02, Tomas Vondra wrote:
> >> On 5/2/22 07:31, Amit Kapila wrote:
> >
> >>> Yeah, or don't allow to define such publications in the first place so
> >>> that different subscriptions can't combine them but I guess that might
> >>> forbid some useful cases as well where publication may not get
> >>> combined with other publications.
> >>
> >> But how would you check that? You don't know which publications will be
> >> combined by a subscription until you create the subscription, right?
> >
Yeah, I was thinking to check for all publications where the same
relation is published but as mentioned that may not be a very good
option as that would unnecessarily block many valid cases.
> > ... and I think this poses a problem: if the publisher has multiple
> > publications and the subscriber later uses those to create a combined
> > subscription, we can check at CREATE SUBSCRIPTION time that they can be
> > combined correctly. But if the publisher decides to change the
> > publications changing the rules and they are no longer consistent, can
> > we throw an error at ALTER PUBLICATION point? If the publisher can
> > detect that they are being used together by some subscription, then
> > maybe we can check consistency in the publication side and everything is
> > all right. But I'm not sure that the publisher knows who is subscribed
> > to what, so this might not be an option.
> >
>
> AFAIK we don't track that (publication/subscription mapping). The
> publications are listed in publication_names parameter of the
> START_REPLICATION command.
>
We don't do that currently but we can as mentioned in my previous
email [1]. Let me write the relevant part again. We need to expose all
publications for a walsender, and then we can find the exact set of
publications where the current publication is used with other
publications and we can check only those publications. So, if we have
three walsenders (walsnd1: pub1, pub2; walsnd2 pub2; walsnd3: pub2,
pub3) in the system and we are currently altering publication pub1
then we need to check only pub3 for any conflicting conditions.
I think it is possible to expose a list of publications for each
walsender as it is stored in each walsenders
LogicalDecodingContext->output_plugin_private. AFAIK, each walsender
can have one such LogicalDecodingContext and we can probably share it
via shared memory?
[1] - https://www.postgresql.org/message-id/CAA4eK1LGX-ig%3D%3DQyL%2B%3D%3DnKvcAS3qFU7%3DNiKL77ukUT-Q_4Xnc...
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-02 11:23 Amit Kapila <[email protected]>
parent: Tomas Vondra <[email protected]>
1 sibling, 1 reply; 58+ messages in thread
From: Amit Kapila @ 2022-05-02 11:23 UTC (permalink / raw)
To: Tomas Vondra <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On Mon, May 2, 2022 at 3:05 PM Tomas Vondra
<[email protected]> wrote:
>
> On 5/2/22 07:31, Amit Kapila wrote:
> > On Mon, May 2, 2022 at 3:27 AM Tomas Vondra
> > <[email protected]> wrote:
> >>
>
> >> The second option has the annoying consequence that it makes this
> >> useless for the "data redaction" use case I described in [2], because
> >> that relies on combining multiple publications.
> >>
> >
> > True, but as a workaround users can create different subscriptions for
> > different publications.
> >
>
> Won't that replicate duplicate data, when the row filters re not
> mutually exclusive?
>
True, but this is a recommendation for mutually exclusive data, and as
far as I can understand the example given by you [1] and Alvaro has
mutually exclusive conditions. In your example, one of the
publications has a condition (region = 'USA') and the other
publication has a condition (region != 'USA'), so will there be a
problem in using different subscriptions for such cases?
[1] - https://www.postgresql.org/message-id/[email protected]
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-02 11:44 Alvaro Herrera <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 3 replies; 58+ messages in thread
From: Alvaro Herrera @ 2022-05-02 11:44 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: Tomas Vondra <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On 2022-May-02, Amit Kapila wrote:
> We don't do that currently but we can as mentioned in my previous
> email [1]. Let me write the relevant part again. We need to expose all
> publications for a walsender, and then we can find the exact set of
> publications where the current publication is used with other
> publications and we can check only those publications. So, if we have
> three walsenders (walsnd1: pub1, pub2; walsnd2 pub2; walsnd3: pub2,
> pub3) in the system and we are currently altering publication pub1
> then we need to check only pub3 for any conflicting conditions.
Hmm ... so what happens in the current system, if you have a running
walsender and modify the publication concurrently? Will the subscriber
start getting the changes with the new publication definition, at some
arbitrary point in the middle of their stream? If that's what we do,
maybe we should have a signalling system which disconnects all
walsenders using that publication, so that they can connect and receive
the new definition.
I don't see anything in the publication DDL that interacts with
walsenders -- perhaps I'm overlooking something.
> I think it is possible to expose a list of publications for each
> walsender as it is stored in each walsenders
> LogicalDecodingContext->output_plugin_private. AFAIK, each walsender
> can have one such LogicalDecodingContext and we can probably share it
> via shared memory?
I guess we need to create a DSM each time a walsender opens a
connection, at START_REPLICATION time. Then ALTER PUBLICATION needs to
connect to all DSMs of all running walsenders and see if they are
reading from it. Is that what you have in mind? Alternatively, we
could have one DSM per publication with a PID array of all walsenders
that are sending it (each walsender needs to add its PID as it starts).
The latter might be better.
--
Álvaro Herrera 48°01'N 7°57'E — https://www.EnterpriseDB.com/
"La rebeldía es la virtud original del hombre" (Arthur Schopenhauer)
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-02 16:30 Alvaro Herrera <[email protected]>
parent: Tomas Vondra <[email protected]>
0 siblings, 0 replies; 58+ messages in thread
From: Alvaro Herrera @ 2022-05-02 16:30 UTC (permalink / raw)
To: Tomas Vondra <[email protected]>; +Cc: Amit Kapila <[email protected]>; Pg Hackers <[email protected]>
On 2022-Apr-28, Tomas Vondra wrote:
> SELECT
> (CASE WHEN (a < 0) OR (a > 0) THEN a ELSE NULL END) AS a,
> (CASE WHEN (a > 0) THEN b ELSE NULL END) AS b,
> (CASE WHEN (a < 0) THEN c ELSE NULL END) AS c
> FROM uno WHERE (a < 0) OR (a > 0)
BTW, looking at the new COPY commands, the idea of "COPY table_foo
(PUBLICATION pub1, pub2)" is looking more and more attractive, as a
replacement for having the replica cons up an ad-hoc subquery to COPY
from. Something to think about for pg16, maybe.
--
Álvaro Herrera Breisgau, Deutschland — https://www.EnterpriseDB.com/
"You're _really_ hosed if the person doing the hiring doesn't understand
relational systems: you end up with a whole raft of programmers, none of
whom has had a Date with the clue stick." (Andrew Sullivan)
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-02 17:36 Tomas Vondra <[email protected]>
parent: Alvaro Herrera <[email protected]>
2 siblings, 1 reply; 58+ messages in thread
From: Tomas Vondra @ 2022-05-02 17:36 UTC (permalink / raw)
To: Alvaro Herrera <[email protected]>; Amit Kapila <[email protected]>; +Cc: Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On 5/2/22 13:44, Alvaro Herrera wrote:
> On 2022-May-02, Amit Kapila wrote:
>
>> We don't do that currently but we can as mentioned in my previous
>> email [1]. Let me write the relevant part again. We need to expose all
>> publications for a walsender, and then we can find the exact set of
>> publications where the current publication is used with other
>> publications and we can check only those publications. So, if we have
>> three walsenders (walsnd1: pub1, pub2; walsnd2 pub2; walsnd3: pub2,
>> pub3) in the system and we are currently altering publication pub1
>> then we need to check only pub3 for any conflicting conditions.
>
> Hmm ... so what happens in the current system, if you have a running
> walsender and modify the publication concurrently? Will the subscriber
> start getting the changes with the new publication definition, at some
> arbitrary point in the middle of their stream? If that's what we do,
> maybe we should have a signalling system which disconnects all
> walsenders using that publication, so that they can connect and receive
> the new definition.
>
> I don't see anything in the publication DDL that interacts with
> walsenders -- perhaps I'm overlooking something.
>
pgoutput.c is relies on relcache callbacks to get notified of changes.
See the stuff that touches replicate_valid and publications_valid. So
the walsender should notice the changes immediately.
Maybe you have some particular case in mind, though?
>> I think it is possible to expose a list of publications for each
>> walsender as it is stored in each walsenders
>> LogicalDecodingContext->output_plugin_private. AFAIK, each walsender
>> can have one such LogicalDecodingContext and we can probably share it
>> via shared memory?
>
> I guess we need to create a DSM each time a walsender opens a
> connection, at START_REPLICATION time. Then ALTER PUBLICATION needs to
> connect to all DSMs of all running walsenders and see if they are
> reading from it. Is that what you have in mind? Alternatively, we
> could have one DSM per publication with a PID array of all walsenders
> that are sending it (each walsender needs to add its PID as it starts).
> The latter might be better.
>
I don't quite follow what we're trying to build here. The walsender
already knows which publications it works with - how else would
pgoutput.c know that? So the walsender should be able to validate the
stuff it's supposed to replicate is OK.
Why would we need to know publications replicated by other walsenders?
And what if the subscriber is not connected at the moment? In that case
there'll be no walsender.
regards
--
Tomas Vondra
EnterpriseDB: http://www.enterprisedb.com
The Enterprise PostgreSQL Company
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-02 17:51 Alvaro Herrera <[email protected]>
parent: Tomas Vondra <[email protected]>
0 siblings, 1 reply; 58+ messages in thread
From: Alvaro Herrera @ 2022-05-02 17:51 UTC (permalink / raw)
To: Tomas Vondra <[email protected]>; +Cc: Amit Kapila <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On 2022-May-02, Tomas Vondra wrote:
> pgoutput.c is relies on relcache callbacks to get notified of changes.
> See the stuff that touches replicate_valid and publications_valid. So
> the walsender should notice the changes immediately.
Hmm, I suppose that makes any changes easy enough to detect. We don't
need a separate signalling mechanism.
But it does mean that the walsender needs to test the consistency of
[rowfilter, column list, published actions] whenever they change for any
of the current publications and it is working for more than one, and
disconnect if the combination no longer complies with the rules. By the
next time the replica tries to connect, START_REPLICATION will throw the
error.
> Why would we need to know publications replicated by other walsenders?
> And what if the subscriber is not connected at the moment? In that case
> there'll be no walsender.
Sure, if the replica is not connected then there's no issue -- as you
say, that replica will fail at START_REPLICATION time.
--
Álvaro Herrera PostgreSQL Developer — https://www.EnterpriseDB.com/
"La gente vulgar sólo piensa en pasar el tiempo;
el que tiene talento, en aprovecharlo"
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-02 18:37 Tomas Vondra <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 0 replies; 58+ messages in thread
From: Tomas Vondra @ 2022-05-02 18:37 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On 5/2/22 13:23, Amit Kapila wrote:
> On Mon, May 2, 2022 at 3:05 PM Tomas Vondra
> <[email protected]> wrote:
>>
>> On 5/2/22 07:31, Amit Kapila wrote:
>>> On Mon, May 2, 2022 at 3:27 AM Tomas Vondra
>>> <[email protected]> wrote:
>>>>
>>
>>>> The second option has the annoying consequence that it makes this
>>>> useless for the "data redaction" use case I described in [2], because
>>>> that relies on combining multiple publications.
>>>>
>>>
>>> True, but as a workaround users can create different subscriptions for
>>> different publications.
>>>
>>
>> Won't that replicate duplicate data, when the row filters re not
>> mutually exclusive?
>>
>
> True, but this is a recommendation for mutually exclusive data, and as
> far as I can understand the example given by you [1] and Alvaro has
> mutually exclusive conditions. In your example, one of the
> publications has a condition (region = 'USA') and the other
> publication has a condition (region != 'USA'), so will there be a
> problem in using different subscriptions for such cases?
>
I kept that example intentionally simple, but I'm sure we could come up
with more complex use cases. Following the "data redaction" idea, we
could also apply the "deny all" approach, and do something like this:
-- replicate the minimal column list by default (replica identity)
CREATE PUBLICATION p1 FOR TABLE t (id, region);
-- replicate more columns for the selected region
CREATE PUBLICATION p2 FOR TABLE t (...) WHERE (region = 'USA')
Now, I admit this is something I just made up, but I think it seems like
a pretty common approach.
regards
--
Tomas Vondra
EnterpriseDB: http://www.enterprisedb.com
The Enterprise PostgreSQL Company
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-02 18:40 Tomas Vondra <[email protected]>
parent: Alvaro Herrera <[email protected]>
0 siblings, 1 reply; 58+ messages in thread
From: Tomas Vondra @ 2022-05-02 18:40 UTC (permalink / raw)
To: Alvaro Herrera <[email protected]>; +Cc: Amit Kapila <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On 5/2/22 19:51, Alvaro Herrera wrote:
> On 2022-May-02, Tomas Vondra wrote:
>
>> pgoutput.c is relies on relcache callbacks to get notified of changes.
>> See the stuff that touches replicate_valid and publications_valid. So
>> the walsender should notice the changes immediately.
>
> Hmm, I suppose that makes any changes easy enough to detect. We don't
> need a separate signalling mechanism.
>
> But it does mean that the walsender needs to test the consistency of
> [rowfilter, column list, published actions] whenever they change for any
> of the current publications and it is working for more than one, and
> disconnect if the combination no longer complies with the rules. By the
> next time the replica tries to connect, START_REPLICATION will throw the
> error.
>
>> Why would we need to know publications replicated by other walsenders?
>> And what if the subscriber is not connected at the moment? In that case
>> there'll be no walsender.
>
> Sure, if the replica is not connected then there's no issue -- as you
> say, that replica will fail at START_REPLICATION time.
>
Right, I got confused a bit.
Anyway, I think the main challenge is defining what exactly we want to
check, in order to ensure "sensible" behavior, without preventing way
too many sensible use cases.
regards
--
Tomas Vondra
EnterpriseDB: http://www.enterprisedb.com
The Enterprise PostgreSQL Company
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-02 20:34 Peter Eisentraut <[email protected]>
2 siblings, 1 reply; 58+ messages in thread
From: Peter Eisentraut @ 2022-05-02 20:34 UTC (permalink / raw)
To: Tomas Vondra <[email protected]>; Amit Kapila <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Pg Hackers <[email protected]>
On 01.05.22 23:42, Tomas Vondra wrote:
> Imagine have a table with customers from different regions, and you want
> to replicate the data somewhere else, but for some reason you can only
> replicate details for one particular region, and subset of columns for
> everyone else. So you'd do something like this:
>
> CREATE PUBLICATION p1 FOR TABLE customers (... all columns ...)
> WHERE region = 'USA';
>
> CREATE PUBLICATION p1 FOR TABLE customers (... subset of columns ...)
> WHERE region != 'USA';
>
> I think ignoring the row filters and just merging the column lists makes
> no sense for this use case.
I'm thinking now the underlying problem is that we shouldn't combine
column lists at all. Examples like the above where you want to redact
values somehow are better addressed with something like triggers or an
actual "column filter" that works dynamically or some other mechanism.
The main purpose, in my mind, of column lists is if the tables
statically have different shapes on publisher and subscriber. Perhaps
for space reasons or regulatory reasons you don't want to replicate
everything. But then it doesn't make sense to combine column lists. If
you decide over here that the subscriber table has this shape and over
there that the subscriber table has that other shape, then the
combination of the two will be a table that has neither shape and so
will not work for anything.
I think in general we should be much more restrictive in how we combine
publications. Unless we are really sure it makes sense, we should
disallow it. Users can always make a new publication with different
settings and subscribe to that directly.
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-03 03:30 Amit Kapila <[email protected]>
parent: Tomas Vondra <[email protected]>
0 siblings, 1 reply; 58+ messages in thread
From: Amit Kapila @ 2022-05-03 03:30 UTC (permalink / raw)
To: Tomas Vondra <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On Tue, May 3, 2022 at 12:10 AM Tomas Vondra
<[email protected]> wrote:
>
> On 5/2/22 19:51, Alvaro Herrera wrote:
> >> Why would we need to know publications replicated by other walsenders?
> >> And what if the subscriber is not connected at the moment? In that case
> >> there'll be no walsender.
> >
> > Sure, if the replica is not connected then there's no issue -- as you
> > say, that replica will fail at START_REPLICATION time.
> >
>
> Right, I got confused a bit.
>
> Anyway, I think the main challenge is defining what exactly we want to
> check, in order to ensure "sensible" behavior, without preventing way
> too many sensible use cases.
>
I could think of below two options:
1. Forbid any case where column list is different for the same table
when combining publications.
2. Forbid if the column list and row filters for a table are different
in the set of publications we are planning to combine. This means we
will allow combining column lists when row filters are not present or
when column list is the same (we don't get anything additional by
combining but the idea is we won't forbid such cases) and row filters
are different.
Now, I think the points in favor of (1) are that the main purpose of
introducing a column list are: (a) the structure/schema of the
subscriber is different from the publisher, (b) want to hide sensitive
columns data. In both cases, it should be fine if we follow (1) and
from Peter E.'s latest email [1] he also seems to be indicating the
same. If we want to be slightly more relaxed then we can probably (2).
We can decide on something else as well but I feel it should be such
that it is easy to explain.
[1] - https://www.postgresql.org/message-id/47dd2cb9-4e96-169f-15ac-f9407fb54d43%40enterprisedb.com
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-03 03:53 Amit Kapila <[email protected]>
parent: Alvaro Herrera <[email protected]>
2 siblings, 0 replies; 58+ messages in thread
From: Amit Kapila @ 2022-05-03 03:53 UTC (permalink / raw)
To: Alvaro Herrera <[email protected]>; +Cc: Tomas Vondra <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On Mon, May 2, 2022 at 6:11 PM Alvaro Herrera <[email protected]> wrote:
>
> On 2022-May-02, Amit Kapila wrote:
>
> > I think it is possible to expose a list of publications for each
> > walsender as it is stored in each walsenders
> > LogicalDecodingContext->output_plugin_private. AFAIK, each walsender
> > can have one such LogicalDecodingContext and we can probably share it
> > via shared memory?
>
> I guess we need to create a DSM each time a walsender opens a
> connection, at START_REPLICATION time. Then ALTER PUBLICATION needs to
> connect to all DSMs of all running walsenders and see if they are
> reading from it. Is that what you have in mind?
>
Yes, something on these lines. We need a way to get the list of
publications each walsender is publishing data for.
> Alternatively, we
> could have one DSM per publication with a PID array of all walsenders
> that are sending it (each walsender needs to add its PID as it starts).
>
I think for this we need to check DSM for all the publications and I
feel in general publications should be more than the number of
walsenders, so the previous approach seems better to me. However, any
one of these or similar ideas should be okay.
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-03 19:40 Tomas Vondra <[email protected]>
parent: Peter Eisentraut <[email protected]>
0 siblings, 1 reply; 58+ messages in thread
From: Tomas Vondra @ 2022-05-03 19:40 UTC (permalink / raw)
To: Peter Eisentraut <[email protected]>; Amit Kapila <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Pg Hackers <[email protected]>
On 5/2/22 22:34, Peter Eisentraut wrote:
> On 01.05.22 23:42, Tomas Vondra wrote:
>> Imagine have a table with customers from different regions, and you want
>> to replicate the data somewhere else, but for some reason you can only
>> replicate details for one particular region, and subset of columns for
>> everyone else. So you'd do something like this:
>>
>> CREATE PUBLICATION p1 FOR TABLE customers (... all columns ...)
>> WHERE region = 'USA';
>>
>> CREATE PUBLICATION p1 FOR TABLE customers (... subset of columns ...)
>> WHERE region != 'USA';
>>
>> I think ignoring the row filters and just merging the column lists makes
>> no sense for this use case.
>
> I'm thinking now the underlying problem is that we shouldn't combine
> column lists at all. Examples like the above where you want to redact
> values somehow are better addressed with something like triggers or an
> actual "column filter" that works dynamically or some other mechanism.
>
So what's wrong with merging the column lists as implemented in the v2
patch, posted a couple days ago?
I don't think triggers are a suitable alternative, as it executes on the
subscriber node. So you have to first copy the data to the remote node,
where it gets filtered. With column filters the data gets redacted on
the publisher.
> The main purpose, in my mind, of column lists is if the tables
> statically have different shapes on publisher and subscriber. Perhaps
> for space reasons or regulatory reasons you don't want to replicate
> everything. But then it doesn't make sense to combine column lists. If
> you decide over here that the subscriber table has this shape and over
> there that the subscriber table has that other shape, then the
> combination of the two will be a table that has neither shape and so
> will not work for anything.
>
Yeah. If we intend to use column lists only to adapt to a different
schema on the subscriber node, then maybe it'd be fine to not merge
column lists. It'd probably be reasonable to allow at least cases with
multiple publications using the same column list, though. In that case
there's no ambiguity.
> I think in general we should be much more restrictive in how we combine
> publications. Unless we are really sure it makes sense, we should
> disallow it. Users can always make a new publication with different
> settings and subscribe to that directly.
I agree with that in principle - correct first, flexibility second. If
the behavior is not correct, it doesn't matter how flexible it is.
I still think the data redaction use case is valid/interesting, but if
we want to impose some restrictions I'm OK with that, as long as it's
done in a way that we can relax in the future to allow that use case
(that is, without introducing any incompatibilities).
However, what's the definition of "correctness" in this context? Without
that it's hard to say if the restrictions make the behavior any more
correct. It'd be unfortunate to impose restritions, which will prevent
some use cases, only to discover we haven't actually made it correct.
For example, is it enough to restrict column lists, or does it need to
restrict e.g. row filters too? And does it need to consider other stuff,
like publications replicating different actions?
For example, if we allow different column lists (or row filters) for
different actions (one publication for insert, another one for update),
we still have the strange behavior described before.
And if we force users to use separate subscriptions, I'm not sure that
really improves the situation for users who actually need that. They'll
do that, and aside from all the problems they'll also face issues with
timing between the two concurrent subscriptions, having to decode stuff
multiple times, etc.
regards
--
Tomas Vondra
EnterpriseDB: http://www.enterprisedb.com
The Enterprise PostgreSQL Company
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-04 13:56 Peter Eisentraut <[email protected]>
parent: Tomas Vondra <[email protected]>
0 siblings, 0 replies; 58+ messages in thread
From: Peter Eisentraut @ 2022-05-04 13:56 UTC (permalink / raw)
To: Tomas Vondra <[email protected]>; Amit Kapila <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Pg Hackers <[email protected]>
On 03.05.22 21:40, Tomas Vondra wrote:
> So what's wrong with merging the column lists as implemented in the v2
> patch, posted a couple days ago?
Merging the column lists is ok if all other publication attributes
match. Otherwise, I think not.
> I don't think triggers are a suitable alternative, as it executes on the
> subscriber node. So you have to first copy the data to the remote node,
> where it gets filtered. With column filters the data gets redacted on
> the publisher.
Right, triggers are not currently a solution. But you could imagine a
redaction filter system that runs on the publisher that modifies rows
before they are sent out.
^ permalink raw reply [nested|flat] 58+ messages in thread
* RE: bogus: logical replication rows/cols combinations
@ 2022-05-06 03:23 [email protected] <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 1 reply; 58+ messages in thread
From: [email protected] @ 2022-05-06 03:23 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; Tomas Vondra <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On Tuesday, May 3, 2022 11:31 AM Amit Kapila <[email protected]> wrote:
>
> On Tue, May 3, 2022 at 12:10 AM Tomas Vondra
> <[email protected]> wrote:
> >
> > On 5/2/22 19:51, Alvaro Herrera wrote:
> > >> Why would we need to know publications replicated by other
> walsenders?
> > >> And what if the subscriber is not connected at the moment? In that case
> > >> there'll be no walsender.
> > >
> > > Sure, if the replica is not connected then there's no issue -- as you
> > > say, that replica will fail at START_REPLICATION time.
> > >
> >
> > Right, I got confused a bit.
> >
> > Anyway, I think the main challenge is defining what exactly we want to
> > check, in order to ensure "sensible" behavior, without preventing way
> > too many sensible use cases.
> >
>
> I could think of below two options:
> 1. Forbid any case where column list is different for the same table
> when combining publications.
> 2. Forbid if the column list and row filters for a table are different
> in the set of publications we are planning to combine. This means we
> will allow combining column lists when row filters are not present or
> when column list is the same (we don't get anything additional by
> combining but the idea is we won't forbid such cases) and row filters
> are different.
>
> Now, I think the points in favor of (1) are that the main purpose of
> introducing a column list are: (a) the structure/schema of the
> subscriber is different from the publisher, (b) want to hide sensitive
> columns data. In both cases, it should be fine if we follow (1) and
> from Peter E.'s latest email [1] he also seems to be indicating the
> same. If we want to be slightly more relaxed then we can probably (2).
> We can decide on something else as well but I feel it should be such
> that it is easy to explain.
I also think it makes sense to add a restriction like (1). I am planning to
implement the restriction if no one objects.
Best regards,
Hou zj
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-06 12:26 Tomas Vondra <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 1 reply; 58+ messages in thread
From: Tomas Vondra @ 2022-05-06 12:26 UTC (permalink / raw)
To: [email protected] <[email protected]>; Amit Kapila <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On 5/6/22 05:23, [email protected] wrote:
> On Tuesday, May 3, 2022 11:31 AM Amit Kapila <[email protected]> wrote:
>>
>> On Tue, May 3, 2022 at 12:10 AM Tomas Vondra
>> <[email protected]> wrote:
>>>
>>> On 5/2/22 19:51, Alvaro Herrera wrote:
>>>>> Why would we need to know publications replicated by other
>> walsenders?
>>>>> And what if the subscriber is not connected at the moment? In that case
>>>>> there'll be no walsender.
>>>>
>>>> Sure, if the replica is not connected then there's no issue -- as you
>>>> say, that replica will fail at START_REPLICATION time.
>>>>
>>>
>>> Right, I got confused a bit.
>>>
>>> Anyway, I think the main challenge is defining what exactly we want to
>>> check, in order to ensure "sensible" behavior, without preventing way
>>> too many sensible use cases.
>>>
>>
>> I could think of below two options:
>> 1. Forbid any case where column list is different for the same table
>> when combining publications.
>> 2. Forbid if the column list and row filters for a table are different
>> in the set of publications we are planning to combine. This means we
>> will allow combining column lists when row filters are not present or
>> when column list is the same (we don't get anything additional by
>> combining but the idea is we won't forbid such cases) and row filters
>> are different.
>>
>> Now, I think the points in favor of (1) are that the main purpose of
>> introducing a column list are: (a) the structure/schema of the
>> subscriber is different from the publisher, (b) want to hide sensitive
>> columns data. In both cases, it should be fine if we follow (1) and
>> from Peter E.'s latest email [1] he also seems to be indicating the
>> same. If we want to be slightly more relaxed then we can probably (2).
>> We can decide on something else as well but I feel it should be such
>> that it is easy to explain.
>
> I also think it makes sense to add a restriction like (1). I am planning to
> implement the restriction if no one objects.
>
I'm not going to block that approach if that's the consensus here,
though I'm not convinced.
Let me point out (1) does *not* work for data redaction use case,
certainly not the example Alvaro and me presented, because that relies
on a combination of row filters and column filters. Requiring all column
lists to be the same (and not specific to row filter) prevents that
example from working. Yes, you can create multiple subscriptions, but
that brings it's own set of challenges too.
I doubt forcing users to use the more complex setup is good idea, and
combining the column lists per [1] seems sound to me.
That being said, the good thing is this restriction seems it might be
relaxed in the future to work per [1], without causing any backwards
compatibility issues.
Should we do something similar for row filters, though? It seems quite
weird we're so concerned about unexpected behavior due to combining
column lists (despite having a patch that makes it behave sanely), and
at the same time wave off similarly strange behavior due to combining
row filters because "that's what you get if you define the publications
in a strange way".
regards
[1]
https://www.postgresql.org/message-id/5a85b8b7-fc1c-364b-5c62-0bb3e1e25824%40enterprisedb.com
--
Tomas Vondra
EnterpriseDB: http://www.enterprisedb.com
The Enterprise PostgreSQL Company
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-06 13:40 Amit Kapila <[email protected]>
parent: Tomas Vondra <[email protected]>
0 siblings, 1 reply; 58+ messages in thread
From: Amit Kapila @ 2022-05-06 13:40 UTC (permalink / raw)
To: Tomas Vondra <[email protected]>; +Cc: [email protected] <[email protected]>; Alvaro Herrera <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On Fri, May 6, 2022 at 5:56 PM Tomas Vondra
<[email protected]> wrote:
>
> >>
> >> I could think of below two options:
> >> 1. Forbid any case where column list is different for the same table
> >> when combining publications.
> >> 2. Forbid if the column list and row filters for a table are different
> >> in the set of publications we are planning to combine. This means we
> >> will allow combining column lists when row filters are not present or
> >> when column list is the same (we don't get anything additional by
> >> combining but the idea is we won't forbid such cases) and row filters
> >> are different.
> >>
> >> Now, I think the points in favor of (1) are that the main purpose of
> >> introducing a column list are: (a) the structure/schema of the
> >> subscriber is different from the publisher, (b) want to hide sensitive
> >> columns data. In both cases, it should be fine if we follow (1) and
> >> from Peter E.'s latest email [1] he also seems to be indicating the
> >> same. If we want to be slightly more relaxed then we can probably (2).
> >> We can decide on something else as well but I feel it should be such
> >> that it is easy to explain.
> >
> > I also think it makes sense to add a restriction like (1). I am planning to
> > implement the restriction if no one objects.
> >
>
> I'm not going to block that approach if that's the consensus here,
> though I'm not convinced.
>
> Let me point out (1) does *not* work for data redaction use case,
> certainly not the example Alvaro and me presented, because that relies
> on a combination of row filters and column filters.
>
This should just forbid the case presented by Alvaro in his first
email in this thread [1].
> Requiring all column
> lists to be the same (and not specific to row filter) prevents that
> example from working. Yes, you can create multiple subscriptions, but
> that brings it's own set of challenges too.
>
> I doubt forcing users to use the more complex setup is good idea, and
> combining the column lists per [1] seems sound to me.
>
> That being said, the good thing is this restriction seems it might be
> relaxed in the future to work per [1], without causing any backwards
> compatibility issues.
>
These are my thoughts as well. Even, if we decide to go via the column
list merging approach (in selective cases), we need to do some
performance testing of that approach as it does much more work per
tuple. It is possible that the impact is not much but still worth
evaluating, so let's try to see the patch to prohibit combining the
column lists then we can decide.
> Should we do something similar for row filters, though? It seems quite
> weird we're so concerned about unexpected behavior due to combining
> column lists (despite having a patch that makes it behave sanely), and
> at the same time wave off similarly strange behavior due to combining
> row filters because "that's what you get if you define the publications
> in a strange way".
>
During development, we found that we can't combine the row-filters for
'insert' and 'update'/'delete' because of replica identity
restrictions, so we have kept them separate. But if we came across
other such things then we can either try to fix those or forbid them.
[1] - https://www.postgresql.org/message-id/202204251548.mudq7jbqnh7r%40alvherre.pgsql
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-06 13:57 Tomas Vondra <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 0 replies; 58+ messages in thread
From: Tomas Vondra @ 2022-05-06 13:57 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; Alvaro Herrera <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On 5/6/22 15:40, Amit Kapila wrote:
> On Fri, May 6, 2022 at 5:56 PM Tomas Vondra
> <[email protected]> wrote:
>>
>>>>
>>>> I could think of below two options:
>>>> 1. Forbid any case where column list is different for the same table
>>>> when combining publications.
>>>> 2. Forbid if the column list and row filters for a table are different
>>>> in the set of publications we are planning to combine. This means we
>>>> will allow combining column lists when row filters are not present or
>>>> when column list is the same (we don't get anything additional by
>>>> combining but the idea is we won't forbid such cases) and row filters
>>>> are different.
>>>>
>>>> Now, I think the points in favor of (1) are that the main purpose of
>>>> introducing a column list are: (a) the structure/schema of the
>>>> subscriber is different from the publisher, (b) want to hide sensitive
>>>> columns data. In both cases, it should be fine if we follow (1) and
>>>> from Peter E.'s latest email [1] he also seems to be indicating the
>>>> same. If we want to be slightly more relaxed then we can probably (2).
>>>> We can decide on something else as well but I feel it should be such
>>>> that it is easy to explain.
>>>
>>> I also think it makes sense to add a restriction like (1). I am planning to
>>> implement the restriction if no one objects.
>>>
>>
>> I'm not going to block that approach if that's the consensus here,
>> though I'm not convinced.
>>
>> Let me point out (1) does *not* work for data redaction use case,
>> certainly not the example Alvaro and me presented, because that relies
>> on a combination of row filters and column filters.
>>
>
> This should just forbid the case presented by Alvaro in his first
> email in this thread [1].
>
>> Requiring all column
>> lists to be the same (and not specific to row filter) prevents that
>> example from working. Yes, you can create multiple subscriptions, but
>> that brings it's own set of challenges too.
>>
>> I doubt forcing users to use the more complex setup is good idea, and
>> combining the column lists per [1] seems sound to me.
>>
>> That being said, the good thing is this restriction seems it might be
>> relaxed in the future to work per [1], without causing any backwards
>> compatibility issues.
>>
>
> These are my thoughts as well. Even, if we decide to go via the column
> list merging approach (in selective cases), we need to do some
> performance testing of that approach as it does much more work per
> tuple. It is possible that the impact is not much but still worth
> evaluating, so let's try to see the patch to prohibit combining the
> column lists then we can decide.
>
Surely we could do some performance testing now. I doubt it's very
expensive - sure, you can construct cases with many row filters / column
lists, but how likely is that in practice?
Moreover, it's not like this would affect existing setups, so even if
it's a bit expensive, we may interpret that as cost of the feature.
>> Should we do something similar for row filters, though? It seems quite
>> weird we're so concerned about unexpected behavior due to combining
>> column lists (despite having a patch that makes it behave sanely), and
>> at the same time wave off similarly strange behavior due to combining
>> row filters because "that's what you get if you define the publications
>> in a strange way".
>>
>
> During development, we found that we can't combine the row-filters for
> 'insert' and 'update'/'delete' because of replica identity
> restrictions, so we have kept them separate. But if we came across
> other such things then we can either try to fix those or forbid them.
>
I understand how we got to the current state. I'm just saying that this
allows defining separate publications for insert, update and delete
actions, and set different row filters for each of them. Which results
in behavior that is hard to explain/understand, especially when it comes
to tablesync.
It seems quite strange to prohibit merging column lists because there
might be some strange behavior that no one described, and allow setups
with different row filters that definitely have strange behavior.
regards
--
Tomas Vondra
EnterpriseDB: http://www.enterprisedb.com
The Enterprise PostgreSQL Company
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-07 05:36 Amit Kapila <[email protected]>
parent: Alvaro Herrera <[email protected]>
2 siblings, 1 reply; 58+ messages in thread
From: Amit Kapila @ 2022-05-07 05:36 UTC (permalink / raw)
To: Alvaro Herrera <[email protected]>; +Cc: Tomas Vondra <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On Mon, May 2, 2022 at 6:11 PM Alvaro Herrera <[email protected]> wrote:
>
> On 2022-May-02, Amit Kapila wrote:
>
>
> > I think it is possible to expose a list of publications for each
> > walsender as it is stored in each walsenders
> > LogicalDecodingContext->output_plugin_private. AFAIK, each walsender
> > can have one such LogicalDecodingContext and we can probably share it
> > via shared memory?
>
> I guess we need to create a DSM each time a walsender opens a
> connection, at START_REPLICATION time. Then ALTER PUBLICATION needs to
> connect to all DSMs of all running walsenders and see if they are
> reading from it. Is that what you have in mind? Alternatively, we
> could have one DSM per publication with a PID array of all walsenders
> that are sending it (each walsender needs to add its PID as it starts).
> The latter might be better.
>
While thinking about using DSM here, I came across one of your commits
f2f9fcb303 which seems to indicate that it is not a good idea to rely
on it but I think you have changed dynamic shared memory to fixed
shared memory usage because that was more suitable rather than DSM is
not portable. Because I see a commit bcbd940806 where we have removed
the 'none' option of dynamic_shared_memory_type. So, I think it should
be okay to use DSM in this context. What do you think?
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-08 18:11 Tomas Vondra <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 1 reply; 58+ messages in thread
From: Tomas Vondra @ 2022-05-08 18:11 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; Alvaro Herrera <[email protected]>; +Cc: Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On 5/7/22 07:36, Amit Kapila wrote:
> On Mon, May 2, 2022 at 6:11 PM Alvaro Herrera <[email protected]> wrote:
>>
>> On 2022-May-02, Amit Kapila wrote:
>>
>>
>>> I think it is possible to expose a list of publications for each
>>> walsender as it is stored in each walsenders
>>> LogicalDecodingContext->output_plugin_private. AFAIK, each walsender
>>> can have one such LogicalDecodingContext and we can probably share it
>>> via shared memory?
>>
>> I guess we need to create a DSM each time a walsender opens a
>> connection, at START_REPLICATION time. Then ALTER PUBLICATION needs to
>> connect to all DSMs of all running walsenders and see if they are
>> reading from it. Is that what you have in mind? Alternatively, we
>> could have one DSM per publication with a PID array of all walsenders
>> that are sending it (each walsender needs to add its PID as it starts).
>> The latter might be better.
>>
>
> While thinking about using DSM here, I came across one of your commits
> f2f9fcb303 which seems to indicate that it is not a good idea to rely
> on it but I think you have changed dynamic shared memory to fixed
> shared memory usage because that was more suitable rather than DSM is
> not portable. Because I see a commit bcbd940806 where we have removed
> the 'none' option of dynamic_shared_memory_type. So, I think it should
> be okay to use DSM in this context. What do you think?
>
Why would any of this be needed?
ALTER PUBLICATION will invalidate the RelationSyncEntry entries in all
walsenders, no? So AFAICS it should be enough to enforce the limitations
in get_rel_sync_entry, which is necessary anyway because the subscriber
may not be connected when ALTER PUBLICATION gets executed.
regards
--
Tomas Vondra
EnterpriseDB: http://www.enterprisedb.com
The Enterprise PostgreSQL Company
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-09 03:45 Amit Kapila <[email protected]>
parent: Tomas Vondra <[email protected]>
0 siblings, 1 reply; 58+ messages in thread
From: Amit Kapila @ 2022-05-09 03:45 UTC (permalink / raw)
To: Tomas Vondra <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On Sun, May 8, 2022 at 11:41 PM Tomas Vondra
<[email protected]> wrote:
>
> On 5/7/22 07:36, Amit Kapila wrote:
> > On Mon, May 2, 2022 at 6:11 PM Alvaro Herrera <[email protected]> wrote:
> >>
> >> On 2022-May-02, Amit Kapila wrote:
> >>
> >>
> >>> I think it is possible to expose a list of publications for each
> >>> walsender as it is stored in each walsenders
> >>> LogicalDecodingContext->output_plugin_private. AFAIK, each walsender
> >>> can have one such LogicalDecodingContext and we can probably share it
> >>> via shared memory?
> >>
> >> I guess we need to create a DSM each time a walsender opens a
> >> connection, at START_REPLICATION time. Then ALTER PUBLICATION needs to
> >> connect to all DSMs of all running walsenders and see if they are
> >> reading from it. Is that what you have in mind? Alternatively, we
> >> could have one DSM per publication with a PID array of all walsenders
> >> that are sending it (each walsender needs to add its PID as it starts).
> >> The latter might be better.
> >>
> >
> > While thinking about using DSM here, I came across one of your commits
> > f2f9fcb303 which seems to indicate that it is not a good idea to rely
> > on it but I think you have changed dynamic shared memory to fixed
> > shared memory usage because that was more suitable rather than DSM is
> > not portable. Because I see a commit bcbd940806 where we have removed
> > the 'none' option of dynamic_shared_memory_type. So, I think it should
> > be okay to use DSM in this context. What do you think?
> >
>
> Why would any of this be needed?
>
> ALTER PUBLICATION will invalidate the RelationSyncEntry entries in all
> walsenders, no? So AFAICS it should be enough to enforce the limitations
> in get_rel_sync_entry,
>
Yes, that should be sufficient to enforce limitations in
get_rel_sync_entry() but it will lead to the following behavior:
a. The Alter Publication command will be successful but later in the
logs, the error will be logged and the user needs to check it and take
appropriate action. Till that time the walsender will be in an error
loop which means it will restart and again lead to the same error till
the user takes some action.
b. As we use historic snapshots, so even after the user takes action
say by changing publication, it won't be reflected. So, the option for
the user would be to drop their subscription.
Am, I missing something? If not, then are we okay with such behavior?
If yes, then I think it would be much easier implementation-wise and
probably advisable at this point. We can document it so that users are
careful and can take necessary action if they get into such a
situation. Any way we can improve this in future as you also suggested
earlier.
> which is necessary anyway because the subscriber
> may not be connected when ALTER PUBLICATION gets executed.
>
If we are not okay with the resultant behavior of detecting this in
get_rel_sync_entry(), then we can solve this in some other way as
Alvaro has indicated in one of his responses which is to detect that
at start replication time probably in the subscriber-side.
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-10 19:05 Tomas Vondra <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 1 reply; 58+ messages in thread
From: Tomas Vondra @ 2022-05-10 19:05 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On 5/9/22 05:45, Amit Kapila wrote:
> On Sun, May 8, 2022 at 11:41 PM Tomas Vondra
> <[email protected]> wrote:
>>
>> On 5/7/22 07:36, Amit Kapila wrote:
>>> On Mon, May 2, 2022 at 6:11 PM Alvaro Herrera <[email protected]> wrote:
>>>>
>>>> On 2022-May-02, Amit Kapila wrote:
>>>>
>>>>
>>>>> I think it is possible to expose a list of publications for each
>>>>> walsender as it is stored in each walsenders
>>>>> LogicalDecodingContext->output_plugin_private. AFAIK, each walsender
>>>>> can have one such LogicalDecodingContext and we can probably share it
>>>>> via shared memory?
>>>>
>>>> I guess we need to create a DSM each time a walsender opens a
>>>> connection, at START_REPLICATION time. Then ALTER PUBLICATION needs to
>>>> connect to all DSMs of all running walsenders and see if they are
>>>> reading from it. Is that what you have in mind? Alternatively, we
>>>> could have one DSM per publication with a PID array of all walsenders
>>>> that are sending it (each walsender needs to add its PID as it starts).
>>>> The latter might be better.
>>>>
>>>
>>> While thinking about using DSM here, I came across one of your commits
>>> f2f9fcb303 which seems to indicate that it is not a good idea to rely
>>> on it but I think you have changed dynamic shared memory to fixed
>>> shared memory usage because that was more suitable rather than DSM is
>>> not portable. Because I see a commit bcbd940806 where we have removed
>>> the 'none' option of dynamic_shared_memory_type. So, I think it should
>>> be okay to use DSM in this context. What do you think?
>>>
>>
>> Why would any of this be needed?
>>
>> ALTER PUBLICATION will invalidate the RelationSyncEntry entries in all
>> walsenders, no? So AFAICS it should be enough to enforce the limitations
>> in get_rel_sync_entry,
>>
>
> Yes, that should be sufficient to enforce limitations in
> get_rel_sync_entry() but it will lead to the following behavior:
> a. The Alter Publication command will be successful but later in the
> logs, the error will be logged and the user needs to check it and take
> appropriate action. Till that time the walsender will be in an error
> loop which means it will restart and again lead to the same error till
> the user takes some action.
> b. As we use historic snapshots, so even after the user takes action
> say by changing publication, it won't be reflected. So, the option for
> the user would be to drop their subscription.
>
> Am, I missing something? If not, then are we okay with such behavior?
> If yes, then I think it would be much easier implementation-wise and
> probably advisable at this point. We can document it so that users are
> careful and can take necessary action if they get into such a
> situation. Any way we can improve this in future as you also suggested
> earlier.
>
>> which is necessary anyway because the subscriber
>> may not be connected when ALTER PUBLICATION gets executed.
>>
>
> If we are not okay with the resultant behavior of detecting this in
> get_rel_sync_entry(), then we can solve this in some other way as
> Alvaro has indicated in one of his responses which is to detect that
> at start replication time probably in the subscriber-side.
>
IMO that behavior is acceptable. We have to do that check anyway, and
the subscription may start failing after ALTER PUBLICATION for a number
of other reasons anyway so the user needs/should check the logs.
And if needed, we can improve this and start doing the proactive-checks
during ALTER PUBLICATION too.
regards
--
Tomas Vondra
EnterpriseDB: http://www.enterprisedb.com
The Enterprise PostgreSQL Company
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-11 03:33 Amit Kapila <[email protected]>
parent: Tomas Vondra <[email protected]>
0 siblings, 1 reply; 58+ messages in thread
From: Amit Kapila @ 2022-05-11 03:33 UTC (permalink / raw)
To: Tomas Vondra <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On Wed, May 11, 2022 at 12:35 AM Tomas Vondra
<[email protected]> wrote:
>
> On 5/9/22 05:45, Amit Kapila wrote:
> > On Sun, May 8, 2022 at 11:41 PM Tomas Vondra
> > <[email protected]> wrote:
> >>
> >> On 5/7/22 07:36, Amit Kapila wrote:
> >>> On Mon, May 2, 2022 at 6:11 PM Alvaro Herrera <[email protected]> wrote:
> >>>>
> >>>> On 2022-May-02, Amit Kapila wrote:
> >>>>
> >>>>
> >>>>> I think it is possible to expose a list of publications for each
> >>>>> walsender as it is stored in each walsenders
> >>>>> LogicalDecodingContext->output_plugin_private. AFAIK, each walsender
> >>>>> can have one such LogicalDecodingContext and we can probably share it
> >>>>> via shared memory?
> >>>>
> >>>> I guess we need to create a DSM each time a walsender opens a
> >>>> connection, at START_REPLICATION time. Then ALTER PUBLICATION needs to
> >>>> connect to all DSMs of all running walsenders and see if they are
> >>>> reading from it. Is that what you have in mind? Alternatively, we
> >>>> could have one DSM per publication with a PID array of all walsenders
> >>>> that are sending it (each walsender needs to add its PID as it starts).
> >>>> The latter might be better.
> >>>>
> >>>
> >>> While thinking about using DSM here, I came across one of your commits
> >>> f2f9fcb303 which seems to indicate that it is not a good idea to rely
> >>> on it but I think you have changed dynamic shared memory to fixed
> >>> shared memory usage because that was more suitable rather than DSM is
> >>> not portable. Because I see a commit bcbd940806 where we have removed
> >>> the 'none' option of dynamic_shared_memory_type. So, I think it should
> >>> be okay to use DSM in this context. What do you think?
> >>>
> >>
> >> Why would any of this be needed?
> >>
> >> ALTER PUBLICATION will invalidate the RelationSyncEntry entries in all
> >> walsenders, no? So AFAICS it should be enough to enforce the limitations
> >> in get_rel_sync_entry,
> >>
> >
> > Yes, that should be sufficient to enforce limitations in
> > get_rel_sync_entry() but it will lead to the following behavior:
> > a. The Alter Publication command will be successful but later in the
> > logs, the error will be logged and the user needs to check it and take
> > appropriate action. Till that time the walsender will be in an error
> > loop which means it will restart and again lead to the same error till
> > the user takes some action.
> > b. As we use historic snapshots, so even after the user takes action
> > say by changing publication, it won't be reflected. So, the option for
> > the user would be to drop their subscription.
> >
> > Am, I missing something? If not, then are we okay with such behavior?
> > If yes, then I think it would be much easier implementation-wise and
> > probably advisable at this point. We can document it so that users are
> > careful and can take necessary action if they get into such a
> > situation. Any way we can improve this in future as you also suggested
> > earlier.
> >
> >> which is necessary anyway because the subscriber
> >> may not be connected when ALTER PUBLICATION gets executed.
> >>
> >
> > If we are not okay with the resultant behavior of detecting this in
> > get_rel_sync_entry(), then we can solve this in some other way as
> > Alvaro has indicated in one of his responses which is to detect that
> > at start replication time probably in the subscriber-side.
> >
>
> IMO that behavior is acceptable.
>
Fair enough, then we should go with a simpler approach to detect it in
pgoutput.c (get_rel_sync_entry).
> We have to do that check anyway, and
> the subscription may start failing after ALTER PUBLICATION for a number
> of other reasons anyway so the user needs/should check the logs.
>
I think ALTER PUBLICATION won't ever lead to failure in walsender.
Sure, users can do something due to which subscriber-side failures can
happen due to constraint failures. Do you have some specific cases in
mind?
> And if needed, we can improve this and start doing the proactive-checks
> during ALTER PUBLICATION too.
>
Agreed.
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 58+ messages in thread
* RE: bogus: logical replication rows/cols combinations
@ 2022-05-11 07:25 [email protected] <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 1 reply; 58+ messages in thread
From: [email protected] @ 2022-05-11 07:25 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; Tomas Vondra <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On Wednesday, May 11, 2022 11:33 AM Amit Kapila <[email protected]> wrote:
> On Wed, May 11, 2022 at 12:35 AM Tomas Vondra
> <[email protected]> wrote:
> >
> > On 5/9/22 05:45, Amit Kapila wrote:
> > > On Sun, May 8, 2022 at 11:41 PM Tomas Vondra
> > > <[email protected]> wrote:
> > >>
> > >> On 5/7/22 07:36, Amit Kapila wrote:
> > >>> On Mon, May 2, 2022 at 6:11 PM Alvaro Herrera
> <[email protected]> wrote:
> > >>>>
> > >>>> On 2022-May-02, Amit Kapila wrote:
> > >>>>
> > >>>>
> > >>>>> I think it is possible to expose a list of publications for each
> > >>>>> walsender as it is stored in each walsenders
> > >>>>> LogicalDecodingContext->output_plugin_private. AFAIK, each
> > >>>>> LogicalDecodingContext->walsender
> > >>>>> can have one such LogicalDecodingContext and we can probably
> > >>>>> share it via shared memory?
> > >>>>
> > >>>> I guess we need to create a DSM each time a walsender opens a
> > >>>> connection, at START_REPLICATION time. Then ALTER PUBLICATION
> > >>>> needs to connect to all DSMs of all running walsenders and see if
> > >>>> they are reading from it. Is that what you have in mind?
> > >>>> Alternatively, we could have one DSM per publication with a PID
> > >>>> array of all walsenders that are sending it (each walsender needs to
> add its PID as it starts).
> > >>>> The latter might be better.
> > >>>>
> > >>>
> > >>> While thinking about using DSM here, I came across one of your
> > >>> commits
> > >>> f2f9fcb303 which seems to indicate that it is not a good idea to
> > >>> rely on it but I think you have changed dynamic shared memory to
> > >>> fixed shared memory usage because that was more suitable rather
> > >>> than DSM is not portable. Because I see a commit bcbd940806 where
> > >>> we have removed the 'none' option of dynamic_shared_memory_type.
> > >>> So, I think it should be okay to use DSM in this context. What do you
> think?
> > >>>
> > >>
> > >> Why would any of this be needed?
> > >>
> > >> ALTER PUBLICATION will invalidate the RelationSyncEntry entries in
> > >> all walsenders, no? So AFAICS it should be enough to enforce the
> > >> limitations in get_rel_sync_entry,
> > >>
> > >
> > > Yes, that should be sufficient to enforce limitations in
> > > get_rel_sync_entry() but it will lead to the following behavior:
> > > a. The Alter Publication command will be successful but later in the
> > > logs, the error will be logged and the user needs to check it and
> > > take appropriate action. Till that time the walsender will be in an
> > > error loop which means it will restart and again lead to the same
> > > error till the user takes some action.
> > > b. As we use historic snapshots, so even after the user takes action
> > > say by changing publication, it won't be reflected. So, the option
> > > for the user would be to drop their subscription.
> > >
> > > Am, I missing something? If not, then are we okay with such behavior?
> > > If yes, then I think it would be much easier implementation-wise and
> > > probably advisable at this point. We can document it so that users
> > > are careful and can take necessary action if they get into such a
> > > situation. Any way we can improve this in future as you also
> > > suggested earlier.
> > >
> > >> which is necessary anyway because the subscriber may not be
> > >> connected when ALTER PUBLICATION gets executed.
> > >>
> > >
> > > If we are not okay with the resultant behavior of detecting this in
> > > get_rel_sync_entry(), then we can solve this in some other way as
> > > Alvaro has indicated in one of his responses which is to detect that
> > > at start replication time probably in the subscriber-side.
> > >
> >
> > IMO that behavior is acceptable.
> >
>
> Fair enough, then we should go with a simpler approach to detect it in
> pgoutput.c (get_rel_sync_entry).
OK, here is the patch that try to check column list in that way. The patch also
check the column list when CREATE SUBSCRIPTION and when starting initial copy.
Best regards,
Hou zj
Attachments:
[application/octet-stream] 0001-Disallow-combining-publication-when-column-list-is-d.patch (20.1K, ../../OS0PR01MB5716A594C58DE4FFD1F8100B94C89@OS0PR01MB5716.jpnprd01.prod.outlook.com/2-0001-Disallow-combining-publication-when-column-list-is-d.patch)
download | inline diff:
From f7f68fd11211ffbe5770fb565ac80620dd6ccea9 Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Wed, 11 May 2022 14:29:05 +0800
Subject: [PATCH] Disallow combining publication when column list is different
for the same table
the main purpose of introducing a column list are statically have different
shapes on publisher and subscriber or hide sensitive columns data. In both
cases, it doesn't seems make sense to combine column lists. So disallow the
cases where column list is different for the same table when combining
publications.
---
src/backend/commands/subscriptioncmds.c | 37 +++++++--
src/backend/replication/logical/tablesync.c | 61 +++++++++-----
src/backend/replication/pgoutput/pgoutput.c | 78 ++++++++---------
src/test/subscription/t/031_column_list.pl | 124 +++++++++-------------------
4 files changed, 154 insertions(+), 146 deletions(-)
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index b94236f..8650cc5 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1753,7 +1753,8 @@ AlterSubscriptionOwner_oid(Oid subid, Oid newOwnerId)
/*
* Get the list of tables which belong to specified publications on the
- * publisher connection.
+ * publisher connection. Also get the column list for each table and check if
+ * column lists are the same in different publications.
*/
static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
@@ -1761,17 +1762,34 @@ fetch_table_list(WalReceiverConn *wrconn, List *publications)
WalRcvExecResult *res;
StringInfoData cmd;
TupleTableSlot *slot;
- Oid tableRow[2] = {TEXTOID, TEXTOID};
+ Oid tableRow[3] = {TEXTOID, TEXTOID, INT2VECTOROID};
List *tablelist = NIL;
initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename\n"
- " FROM pg_catalog.pg_publication_tables t\n"
+ appendStringInfoString(&cmd,
+ "SELECT DISTINCT t.schemaname,\n"
+ " t.tablename,\n"
+ " (CASE WHEN (array_length(pr.prattrs, 1) = t.relnatts)\n"
+ " THEN NULL ELSE pr.prattrs END)\n"
+ " FROM (SELECT P.pubname AS pubname,\n"
+ " N.nspname AS schemaname,\n"
+ " C.relname AS tablename,\n"
+ " P.oid AS pubid,\n"
+ " C.oid AS reloid,\n"
+ " C.relnatts\n"
+ " FROM pg_publication P,\n"
+ " LATERAL pg_get_publication_tables(P.pubname) GPT,\n"
+ " pg_class C JOIN pg_namespace N\n"
+ " ON (N.oid = C.relnamespace)\n"
+ " WHERE C.oid = GPT.relid) t\n"
+ " LEFT OUTER JOIN pg_publication_rel pr\n"
+ " ON (t.pubid = pr.prpubid AND\n"
+ " pr.prrelid = reloid)\n"
" WHERE t.pubname IN (");
get_publications_str(publications, &cmd, true);
appendStringInfoChar(&cmd, ')');
- res = walrcv_exec(wrconn, cmd.data, 2, tableRow);
+ res = walrcv_exec(wrconn, cmd.data, 3, tableRow);
pfree(cmd.data);
if (res->status != WALRCV_OK_TUPLES)
@@ -1795,7 +1813,14 @@ fetch_table_list(WalReceiverConn *wrconn, List *publications)
Assert(!isnull);
rv = makeRangeVar(nspname, relname, -1);
- tablelist = lappend(tablelist, rv);
+
+ if (list_member(tablelist, rv))
+ ereport(WARNING,
+ errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot use different column lists for table \"%s.%s\" in different publications",
+ nspname, relname));
+ else
+ tablelist = lappend(tablelist, rv);
ExecClearTuple(slot);
}
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 49ceec3..8ac4171 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -771,7 +771,7 @@ fetch_remote_table_info(char *nspname, char *relname,
{
WalRcvExecResult *pubres;
TupleTableSlot *slot;
- Oid attrsRow[] = {INT2OID};
+ Oid attrsRow[] = {INT2VECTOROID};
StringInfoData pub_names;
bool first = true;
@@ -786,21 +786,20 @@ fetch_remote_table_info(char *nspname, char *relname,
/*
* Fetch info about column lists for the relation (from all the
- * publications). We unnest the int2vector values, because that
- * makes it easier to combine lists by simply adding the attnums
- * to a new bitmap (without having to parse the int2vector data).
- * This preserves NULL values, so that if one of the publications
- * has no column list, we'll know that.
+ * publications).
*/
resetStringInfo(&cmd);
appendStringInfo(&cmd,
- "SELECT DISTINCT unnest"
+ "SELECT DISTINCT"
+ " (CASE WHEN (array_length(pr.prattrs, 1) = c.relnatts)"
+ " THEN NULL ELSE pr.prattrs END)"
" FROM pg_publication p"
" LEFT OUTER JOIN pg_publication_rel pr"
" ON (p.oid = pr.prpubid AND pr.prrelid = %u)"
" LEFT OUTER JOIN unnest(pr.prattrs) ON TRUE,"
- " LATERAL pg_get_publication_tables(p.pubname) gpt"
- " WHERE gpt.relid = %u"
+ " LATERAL pg_get_publication_tables(p.pubname) gpt,"
+ " pg_class c"
+ " WHERE gpt.relid = %u AND c.oid = gpt.relid"
" AND p.pubname IN ( %s )",
lrel->remoteid,
lrel->remoteid,
@@ -815,27 +814,49 @@ fetch_remote_table_info(char *nspname, char *relname,
errmsg("could not fetch column list info for table \"%s.%s\" from publisher: %s",
nspname, relname, pubres->err)));
+ first = true;
+
/*
- * Merge the column lists (from different publications) by creating
- * a single bitmap with all the attnums. If we find a NULL value,
- * that means one of the publications has no column list for the
- * table we're syncing.
+ * Traverse the column lists from different publications and build a
+ * single bitmap with the attnums.
+ *
+ * During the loop, check that if all the column lists are the same and
+ * report an error if not.
+ *
+ * If we find a NULL value, that means one of the publications has no
+ * column list for the table we're syncing.
*/
slot = MakeSingleTupleTableSlot(pubres->tupledesc, &TTSOpsMinimalTuple);
while (tuplestore_gettupleslot(pubres->tuplestore, true, false, slot))
{
Datum cfval = slot_getattr(slot, 1, &isnull);
+ Bitmapset *cols = NULL;
- /* NULL means empty column list, so we're done. */
- if (isnull)
+ if (!isnull)
{
- bms_free(included_cols);
- included_cols = NULL;
- break;
+ ArrayType *arr;
+ int nelems;
+ int16 *elems;
+
+ arr = DatumGetArrayTypeP(cfval);
+ nelems = ARR_DIMS(arr)[0];
+ elems = (int16 *) ARR_DATA_PTR(arr);
+
+ for (int i = 0; i < nelems; i++)
+ cols = bms_add_member(cols, elems[i]);
}
- included_cols = bms_add_member(included_cols,
- DatumGetInt16(cfval));
+ /* NULL means empty column list. */
+ if (first)
+ {
+ included_cols = cols;
+ first = false;
+ }
+ else if (!bms_equal(included_cols, cols))
+ ereport(ERROR,
+ errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot use different column lists for table \"%s.%s\" in different publications",
+ nspname, relname));
ExecClearTuple(slot);
}
diff --git a/src/backend/replication/pgoutput/pgoutput.c b/src/backend/replication/pgoutput/pgoutput.c
index b197bfd..7ca09de 100644
--- a/src/backend/replication/pgoutput/pgoutput.c
+++ b/src/backend/replication/pgoutput/pgoutput.c
@@ -803,16 +803,13 @@ pgoutput_row_filter_exec_expr(ExprState *state, ExprContext *econtext)
* Make sure the per-entry memory context exists.
*/
static void
-pgoutput_ensure_entry_cxt(PGOutputData *data, RelationSyncEntry *entry)
+pgoutput_ensure_entry_cxt(PGOutputData *data, RelationSyncEntry *entry,
+ Relation relation)
{
- Relation relation;
-
/* The context may already exist, in which case bail out. */
if (entry->entry_cxt)
return;
- relation = RelationIdGetRelation(entry->publish_as_relid);
-
entry->entry_cxt = AllocSetContextCreate(data->cachectx,
"entry private context",
ALLOCSET_SMALL_SIZES);
@@ -941,7 +938,7 @@ pgoutput_row_filter_init(PGOutputData *data, List *publications,
{
Relation relation = RelationIdGetRelation(entry->publish_as_relid);
- pgoutput_ensure_entry_cxt(data, entry);
+ pgoutput_ensure_entry_cxt(data, entry, relation);
/*
* Now all the filters for all pubactions are known. Combine them when
@@ -978,14 +975,20 @@ pgoutput_column_list_init(PGOutputData *data, List *publications,
RelationSyncEntry *entry)
{
ListCell *lc;
+ bool first = true;
+ Relation relation = RelationIdGetRelation(entry->publish_as_relid);
/*
* Find if there are any column lists for this relation. If there are,
- * build a bitmap merging all the column lists.
+ * build a bitmap using the column lists.
*
- * All the given publication-table mappings must be checked.
+ * Note that we don't support the case where column list is different for
+ * the same table when combining publications. But we still need to check
+ * all the given publication-table mappings and report an error if any
+ * publications have different column list.
*
- * Multiple publications might have multiple column lists for this relation.
+ * Multiple publications might have multiple column lists for this
+ * relation.
*
* FOR ALL TABLES and FOR ALL TABLES IN SCHEMA implies "don't use column
* list" so it takes precedence.
@@ -995,12 +998,7 @@ pgoutput_column_list_init(PGOutputData *data, List *publications,
Publication *pub = lfirst(lc);
HeapTuple cftuple = NULL;
Datum cfdatum = 0;
-
- /*
- * Assume there's no column list. Only if we find pg_publication_rel
- * entry with a column list we'll switch it to false.
- */
- bool pub_no_list = true;
+ Bitmapset *cols = NULL;
/*
* If the publication is FOR ALL TABLES then it is treated the same as if
@@ -1008,6 +1006,8 @@ pgoutput_column_list_init(PGOutputData *data, List *publications,
*/
if (!pub->alltables)
{
+ bool pub_no_list = true;
+
/*
* Check for the presence of a column list in this publication.
*
@@ -1033,39 +1033,43 @@ pgoutput_column_list_init(PGOutputData *data, List *publications,
/*
* Build the column list bitmap in the per-entry context.
- *
- * We need to merge column lists from all publications, so we
- * update the same bitmapset. If the column list is null, we
- * interpret it as replicating all columns.
*/
if (!pub_no_list) /* when not null */
{
- pgoutput_ensure_entry_cxt(data, entry);
+ pgoutput_ensure_entry_cxt(data, entry, relation);
+
+ cols = pub_collist_to_bitmapset(cols, cfdatum,
+ entry->entry_cxt);
- entry->columns = pub_collist_to_bitmapset(entry->columns,
- cfdatum,
- entry->entry_cxt);
+ /*
+ * If column list includes all the columns of the table,
+ * set it to NULL.
+ */
+ if (bms_num_members(cols) == RelationGetNumberOfAttributes(relation))
+ {
+ bms_free(cols);
+ cols = NULL;
+ }
}
+
+ ReleaseSysCache(cftuple);
}
}
- /*
- * Found a publication with no column list, so we're done. But first
- * discard column list we might have from preceding publications.
- */
- if (pub_no_list)
+ if (first)
{
- if (cftuple)
- ReleaseSysCache(cftuple);
-
- bms_free(entry->columns);
- entry->columns = NULL;
-
- break;
+ entry->columns = cols;
+ first = false;
}
-
- ReleaseSysCache(cftuple);
+ else if (!bms_equal(entry->columns, cols))
+ ereport(ERROR,
+ errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot use different column lists for table \"%s.%s\" in different publications",
+ get_namespace_name(RelationGetNamespace(relation)),
+ RelationGetRelationName(relation)));
} /* loop all subscribed publications */
+
+ RelationClose(relation);
}
/*
diff --git a/src/test/subscription/t/031_column_list.pl b/src/test/subscription/t/031_column_list.pl
index bdcf3e4..3dfe7d1 100644
--- a/src/test/subscription/t/031_column_list.pl
+++ b/src/test/subscription/t/031_column_list.pl
@@ -21,6 +21,8 @@ $node_subscriber->start;
my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres';
+my $offset = 0;
+
sub wait_for_subscription_sync
{
my ($node) = @_;
@@ -334,12 +336,12 @@ is($result, qq(1|abc
2|xyz), 'update on column tab2.c is not replicated');
-# TEST: add a table to two publications with different column lists, and
+# TEST: add a table to two publications with same column lists, and
# create a single subscription replicating both publications
$node_publisher->safe_psql('postgres', qq(
CREATE TABLE tab5 (a int PRIMARY KEY, b int, c int, d int);
CREATE PUBLICATION pub2 FOR TABLE tab5 (a, b);
- CREATE PUBLICATION pub3 FOR TABLE tab5 (a, d);
+ CREATE PUBLICATION pub3 FOR TABLE tab5 (a, b);
-- insert a couple initial records
INSERT INTO tab5 VALUES (1, 11, 111, 1111);
@@ -358,8 +360,7 @@ wait_for_subscription_sync($node_subscriber);
$node_publisher->wait_for_catchup('sub1');
-# insert data and make sure all the columns (union of the columns lists)
-# get fully replicated
+# insert data and make sure all the columns get fully replicated
$node_publisher->safe_psql('postgres', qq(
INSERT INTO tab5 VALUES (3, 33, 333, 3333);
INSERT INTO tab5 VALUES (4, 44, 444, 4444);
@@ -368,39 +369,11 @@ $node_publisher->safe_psql('postgres', qq(
$node_publisher->wait_for_catchup('sub1');
is($node_subscriber->safe_psql('postgres',"SELECT * FROM tab5 ORDER BY a"),
- qq(1|11|1111
-2|22|2222
-3|33|3333
-4|44|4444),
- 'overlapping publications with overlapping column lists');
-
-# and finally, remove the column list for one of the publications, which
-# means replicating all columns (removing the column list), but first add
-# the missing column to the table on subscriber
-$node_publisher->safe_psql('postgres', qq(
- ALTER PUBLICATION pub3 SET TABLE tab5;
-));
-
-$node_subscriber->safe_psql('postgres', qq(
- ALTER SUBSCRIPTION sub1 REFRESH PUBLICATION;
- ALTER TABLE tab5 ADD COLUMN c INT;
-));
-
-wait_for_subscription_sync($node_subscriber);
-
-$node_publisher->safe_psql('postgres', qq(
- INSERT INTO tab5 VALUES (5, 55, 555, 5555);
-));
-
-$node_publisher->wait_for_catchup('sub1');
-
-is($node_subscriber->safe_psql('postgres',"SELECT * FROM tab5 ORDER BY a"),
- qq(1|11|1111|
-2|22|2222|
-3|33|3333|
-4|44|4444|
-5|55|5555|555),
- 'overlapping publications with overlapping column lists');
+ qq(1|11|
+2|22|
+3|33|
+4|44|),
+ 'insert on column tab5.d is not replicated');
# TEST: create a table with a column list, then change the replica
# identity by replacing a primary key (but use a different column in
@@ -822,51 +795,18 @@ is($node_subscriber->safe_psql('postgres',"SELECT * FROM test_part_d ORDER BY a,
3|),
'partitions with different replica identities not replicated correctly');
-# TEST: With a table included in multiple publications, we should use a
-# union of the column lists. So with column lists (a,b) and (a,c) we
-# should replicate (a,b,c).
-
-$node_publisher->safe_psql('postgres', qq(
- CREATE TABLE test_mix_1 (a int PRIMARY KEY, b int, c int);
- CREATE PUBLICATION pub_mix_1 FOR TABLE test_mix_1 (a, b);
- CREATE PUBLICATION pub_mix_2 FOR TABLE test_mix_1 (a, c);
-
- -- initial data
- INSERT INTO test_mix_1 VALUES (1, 2, 3);
-));
-
-$node_subscriber->safe_psql('postgres', qq(
- CREATE TABLE test_mix_1 (a int PRIMARY KEY, b int, c int);
- ALTER SUBSCRIPTION sub1 SET PUBLICATION pub_mix_1, pub_mix_2;
-));
-
-wait_for_subscription_sync($node_subscriber);
-
-$node_publisher->safe_psql('postgres', qq(
- INSERT INTO test_mix_1 VALUES (4, 5, 6);
-));
-
-$node_publisher->wait_for_catchup('sub1');
-
-is($node_subscriber->safe_psql('postgres',"SELECT * FROM test_mix_1 ORDER BY a"),
- qq(1|2|3
-4|5|6),
- 'a mix of publications should use a union of column list');
-
-
-# TEST: With a table included in multiple publications, we should use a
-# union of the column lists. If any of the publications is FOR ALL
-# TABLES, we should replicate all columns.
+# TEST: With a table included in the publications is FOR ALL TABLES, it means
+# replicate all columns.
# drop unnecessary tables, so as not to interfere with the FOR ALL TABLES
$node_publisher->safe_psql('postgres', qq(
- DROP TABLE tab1, tab2, tab3, tab4, tab5, tab6, tab7, test_mix_1,
+ DROP TABLE tab1, tab2, tab3, tab4, tab5, tab6, tab7,
test_part, test_part_a, test_part_b, test_part_c, test_part_d;
));
$node_publisher->safe_psql('postgres', qq(
CREATE TABLE test_mix_2 (a int PRIMARY KEY, b int, c int);
- CREATE PUBLICATION pub_mix_3 FOR TABLE test_mix_2 (a, b);
+ CREATE PUBLICATION pub_mix_3 FOR TABLE test_mix_2 (a, b, c);
CREATE PUBLICATION pub_mix_4 FOR ALL TABLES;
-- initial data
@@ -890,12 +830,11 @@ $node_publisher->wait_for_catchup('sub1');
is($node_subscriber->safe_psql('postgres',"SELECT * FROM test_mix_2"),
qq(1|2|3
4|5|6),
- 'a mix of publications should use a union of column list');
+ 'all columns should be replicated');
-# TEST: With a table included in multiple publications, we should use a
-# union of the column lists. If any of the publications is FOR ALL
-# TABLES IN SCHEMA, we should replicate all columns.
+# TEST: With a table included in the publication which is FOR ALL TABLES IN
+# SCHEMA, it means replicate all columns.
$node_subscriber->safe_psql('postgres', qq(
DROP SUBSCRIPTION sub1;
@@ -905,7 +844,7 @@ $node_subscriber->safe_psql('postgres', qq(
$node_publisher->safe_psql('postgres', qq(
DROP TABLE test_mix_2;
CREATE TABLE test_mix_3 (a int PRIMARY KEY, b int, c int);
- CREATE PUBLICATION pub_mix_5 FOR TABLE test_mix_3 (a, b);
+ CREATE PUBLICATION pub_mix_5 FOR TABLE test_mix_3 (a, b, c);
CREATE PUBLICATION pub_mix_6 FOR ALL TABLES IN SCHEMA public;
-- initial data
@@ -927,8 +866,7 @@ $node_publisher->wait_for_catchup('sub1');
is($node_subscriber->safe_psql('postgres',"SELECT * FROM test_mix_3"),
qq(1|2|3
4|5|6),
- 'a mix of publications should use a union of column list');
-
+ 'all columns should be replicated');
# TEST: Check handling of publish_via_partition_root - if a partition is
# published through partition root, we should only apply the column list
@@ -979,7 +917,7 @@ is($node_subscriber->safe_psql('postgres',"SELECT * FROM test_root ORDER BY a, b
# TEST: Multiple publications which publish schema of parent table and
# partition. The partition is published through two publications, once
# through a schema (so no column list) containing the parent, and then
-# also directly (with a columns list). The expected outcome is there is
+# also directly (all columns). The expected outcome is there is
# no column list.
$node_publisher->safe_psql('postgres', qq(
@@ -990,7 +928,7 @@ $node_publisher->safe_psql('postgres', qq(
CREATE TABLE t_1 PARTITION OF s1.t FOR VALUES FROM (1) TO (10);
CREATE PUBLICATION pub1 FOR ALL TABLES IN SCHEMA s1;
- CREATE PUBLICATION pub2 FOR TABLE t_1(b);
+ CREATE PUBLICATION pub2 FOR TABLE t_1(a, b, c);
-- initial data
INSERT INTO s1.t VALUES (1, 2, 3);
@@ -1124,6 +1062,26 @@ is($node_subscriber->safe_psql('postgres',"SELECT * FROM t ORDER BY a, b, c"),
4||),
'publication containing both parent and child relation');
+# TEST: With a table included in multiple publications with different column
+# lists, we should catch the error in the log
+
+$node_publisher->safe_psql('postgres', qq(
+ CREATE TABLE test_mix_1 (a int PRIMARY KEY, b int, c int);
+ CREATE PUBLICATION pub_mix_1 FOR TABLE test_mix_1 (a, b);
+ CREATE PUBLICATION pub_mix_2 FOR TABLE test_mix_1 (a, c);
+));
+
+$node_subscriber->safe_psql('postgres', qq(
+ CREATE TABLE test_mix_1 (a int PRIMARY KEY, b int, c int);
+ ALTER SUBSCRIPTION sub1 SET PUBLICATION pub_mix_1, pub_mix_2 WITH (copy_data = false);
+));
+
+$node_publisher->wait_for_catchup('sub1');
+
+my $logfile = slurp_file($node_subscriber->logfile, $offset);
+ok( $logfile =~
+ qr/cannot use different column lists for table "public.test_mix_1" in different publications/,
+ 'different column lists detected');
$node_subscriber->stop('fast');
$node_publisher->stop('fast');
--
2.7.2.windows.1
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-12 06:45 Amit Kapila <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 2 replies; 58+ messages in thread
From: Amit Kapila @ 2022-05-12 06:45 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Tomas Vondra <[email protected]>; Alvaro Herrera <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On Wed, May 11, 2022 at 12:55 PM [email protected]
<[email protected]> wrote:
>
> On Wednesday, May 11, 2022 11:33 AM Amit Kapila <[email protected]> wrote:
> >
> > Fair enough, then we should go with a simpler approach to detect it in
> > pgoutput.c (get_rel_sync_entry).
>
> OK, here is the patch that try to check column list in that way. The patch also
> check the column list when CREATE SUBSCRIPTION and when starting initial copy.
>
Few comments:
===============
1.
initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename\n"
- " FROM pg_catalog.pg_publication_tables t\n"
+ appendStringInfoString(&cmd,
+ "SELECT DISTINCT t.schemaname,\n"
+ " t.tablename,\n"
+ " (CASE WHEN (array_length(pr.prattrs, 1) = t.relnatts)\n"
+ " THEN NULL ELSE pr.prattrs END)\n"
+ " FROM (SELECT P.pubname AS pubname,\n"
+ " N.nspname AS schemaname,\n"
+ " C.relname AS tablename,\n"
+ " P.oid AS pubid,\n"
+ " C.oid AS reloid,\n"
+ " C.relnatts\n"
+ " FROM pg_publication P,\n"
+ " LATERAL pg_get_publication_tables(P.pubname) GPT,\n"
+ " pg_class C JOIN pg_namespace N\n"
+ " ON (N.oid = C.relnamespace)\n"
+ " WHERE C.oid = GPT.relid) t\n"
+ " LEFT OUTER JOIN pg_publication_rel pr\n"
+ " ON (t.pubid = pr.prpubid AND\n"
+ " pr.prrelid = reloid)\n"
Can we modify pg_publication_tables to get the row filter and column
list and then use it directly instead of constructing this query?
2.
+ if (list_member(tablelist, rv))
+ ereport(WARNING,
+ errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot use different column lists for table \"%s.%s\" in
different publications",
+ nspname, relname));
+ else
Can we write comments to explain why we are using WARNING here instead of ERROR?
3.
static void
-pgoutput_ensure_entry_cxt(PGOutputData *data, RelationSyncEntry *entry)
+pgoutput_ensure_entry_cxt(PGOutputData *data, RelationSyncEntry *entry,
+ Relation relation)
What is the need to change this interface as part of this patch?
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-12 08:32 Amit Kapila <[email protected]>
parent: Amit Kapila <[email protected]>
1 sibling, 0 replies; 58+ messages in thread
From: Amit Kapila @ 2022-05-12 08:32 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Tomas Vondra <[email protected]>; Alvaro Herrera <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On Thu, May 12, 2022 at 12:15 PM Amit Kapila <[email protected]> wrote:
>
> On Wed, May 11, 2022 at 12:55 PM [email protected]
> <[email protected]> wrote:
> >
> > On Wednesday, May 11, 2022 11:33 AM Amit Kapila <[email protected]> wrote:
> > >
> > > Fair enough, then we should go with a simpler approach to detect it in
> > > pgoutput.c (get_rel_sync_entry).
> >
> > OK, here is the patch that try to check column list in that way. The patch also
> > check the column list when CREATE SUBSCRIPTION and when starting initial copy.
> >
>
> Few comments:
> ===============
...
One more point, I think we should update the docs for this.
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 58+ messages in thread
* RE: bogus: logical replication rows/cols combinations
@ 2022-05-13 06:02 [email protected] <[email protected]>
parent: Amit Kapila <[email protected]>
1 sibling, 1 reply; 58+ messages in thread
From: [email protected] @ 2022-05-13 06:02 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: Tomas Vondra <[email protected]>; Alvaro Herrera <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On Thursday, May 12, 2022 2:45 PM Amit Kapila <[email protected]> wrote:
>
> On Wed, May 11, 2022 at 12:55 PM [email protected]
> <[email protected]> wrote:
> >
> > On Wednesday, May 11, 2022 11:33 AM Amit Kapila
> <[email protected]> wrote:
> > >
> > > Fair enough, then we should go with a simpler approach to detect it
> > > in pgoutput.c (get_rel_sync_entry).
> >
> > OK, here is the patch that try to check column list in that way. The
> > patch also check the column list when CREATE SUBSCRIPTION and when
> starting initial copy.
> >
>
> Few comments:
> ===============
> 1.
> initStringInfo(&cmd);
> - appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname,
> t.tablename\n"
> - " FROM pg_catalog.pg_publication_tables t\n"
> + appendStringInfoString(&cmd,
> + "SELECT DISTINCT t.schemaname,\n"
> + " t.tablename,\n"
> + " (CASE WHEN (array_length(pr.prattrs, 1) = t.relnatts)\n"
> + " THEN NULL ELSE pr.prattrs END)\n"
> + " FROM (SELECT P.pubname AS pubname,\n"
> + " N.nspname AS schemaname,\n"
> + " C.relname AS tablename,\n"
> + " P.oid AS pubid,\n"
> + " C.oid AS reloid,\n"
> + " C.relnatts\n"
> + " FROM pg_publication P,\n"
> + " LATERAL pg_get_publication_tables(P.pubname) GPT,\n"
> + " pg_class C JOIN pg_namespace N\n"
> + " ON (N.oid = C.relnamespace)\n"
> + " WHERE C.oid = GPT.relid) t\n"
> + " LEFT OUTER JOIN pg_publication_rel pr\n"
> + " ON (t.pubid = pr.prpubid AND\n"
> + " pr.prrelid = reloid)\n"
>
> Can we modify pg_publication_tables to get the row filter and column list and
> then use it directly instead of constructing this query?
Agreed. If we can get columnlist and rowfilter from pg_publication_tables, it
will be more convenient. And I think users that want to fetch the columnlist
and rowfilter of table can also benefit from it.
> 2.
> + if (list_member(tablelist, rv))
> + ereport(WARNING,
> + errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
> + errmsg("cannot use different column lists for table \"%s.%s\" in
> different publications",
> + nspname, relname));
> + else
>
> Can we write comments to explain why we are using WARNING here instead of
> ERROR?
>
> 3.
> static void
> -pgoutput_ensure_entry_cxt(PGOutputData *data, RelationSyncEntry *entry)
> +pgoutput_ensure_entry_cxt(PGOutputData *data, RelationSyncEntry *entry,
> + Relation relation)
>
> What is the need to change this interface as part of this patch?
Attach the new version patch which addressed these comments and update the
document. 0001 patch is to extent the view and 0002 patch is to add restriction
for column list.
Best regards,
Hou zj
Attachments:
[application/octet-stream] 0002-Disallow-combining-publication-when-column-list-is-d.patch (18.2K, ../../OS0PR01MB571698DEFCE9ACC6DC40510794CA9@OS0PR01MB5716.jpnprd01.prod.outlook.com/2-0002-Disallow-combining-publication-when-column-list-is-d.patch)
download | inline diff:
From e1fb8c867353262a43419ee1892bbd64f4d613d7 Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Fri, 13 May 2022 13:19:49 +0800
Subject: [PATCH] Disallow combining publication when column list is different
for the same table
the main purpose of introducing a column list are statically have different
shapes on publisher and subscriber or hide sensitive columns data. In both
cases, it doesn't seems make sense to combine column lists. So disallow the
cases where column list is different for the same table when combining
publications.
---
src/backend/commands/subscriptioncmds.c | 24 ++++--
src/backend/replication/logical/tablesync.c | 61 ++++++++-----
src/backend/replication/pgoutput/pgoutput.c | 75 ++++++++--------
src/test/subscription/t/031_column_list.pl | 127 +++++++++-------------------
4 files changed, 137 insertions(+), 150 deletions(-)
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 690cdaa..c19eb81 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1753,7 +1753,8 @@ AlterSubscriptionOwner_oid(Oid subid, Oid newOwnerId)
/*
* Get the list of tables which belong to specified publications on the
- * publisher connection.
+ * publisher connection. Also get the column list for each table and check if
+ * column lists are the same in different publications.
*/
static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
@@ -1761,17 +1762,18 @@ fetch_table_list(WalReceiverConn *wrconn, List *publications)
WalRcvExecResult *res;
StringInfoData cmd;
TupleTableSlot *slot;
- Oid tableRow[2] = {TEXTOID, TEXTOID};
+ Oid tableRow[3] = {TEXTOID, TEXTOID, INT2VECTOROID};
List *tablelist = NIL;
initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename\n"
+ appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename, \n"
+ " t.columnlist\n"
" FROM pg_catalog.pg_publication_tables t\n"
" WHERE t.pubname IN (");
get_publications_str(publications, &cmd, true);
appendStringInfoChar(&cmd, ')');
- res = walrcv_exec(wrconn, cmd.data, 2, tableRow);
+ res = walrcv_exec(wrconn, cmd.data, 3, tableRow);
pfree(cmd.data);
if (res->status != WALRCV_OK_TUPLES)
@@ -1795,7 +1797,19 @@ fetch_table_list(WalReceiverConn *wrconn, List *publications)
Assert(!isnull);
rv = makeRangeVar(nspname, relname, -1);
- tablelist = lappend(tablelist, rv);
+
+ /*
+ * We only throw a warning here so that the subcription can still be
+ * created and let user aware that something is going to fail later and
+ * they can fix the publications afterwards.
+ */
+ if (list_member(tablelist, rv))
+ ereport(WARNING,
+ errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot use different column lists for table \"%s.%s\" in different publications",
+ nspname, relname));
+ else
+ tablelist = lappend(tablelist, rv);
ExecClearTuple(slot);
}
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 994c7a0..42de832 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -771,7 +771,7 @@ fetch_remote_table_info(char *nspname, char *relname,
{
WalRcvExecResult *pubres;
TupleTableSlot *slot;
- Oid attrsRow[] = {INT2OID};
+ Oid attrsRow[] = {INT2VECTOROID};
StringInfoData pub_names;
bool first = true;
@@ -786,19 +786,17 @@ fetch_remote_table_info(char *nspname, char *relname,
/*
* Fetch info about column lists for the relation (from all the
- * publications). We unnest the int2vector values, because that makes
- * it easier to combine lists by simply adding the attnums to a new
- * bitmap (without having to parse the int2vector data). This
- * preserves NULL values, so that if one of the publications has no
- * column list, we'll know that.
+ * publications).
*/
resetStringInfo(&cmd);
appendStringInfo(&cmd,
- "SELECT DISTINCT unnest"
+ "SELECT DISTINCT"
+ " (CASE WHEN (array_length(gpt.attrs, 1) = c.relnatts)"
+ " THEN NULL ELSE gpt.attrs END)"
" FROM pg_publication p,"
- " LATERAL pg_get_publication_tables(p.pubname) gpt"
- " LEFT OUTER JOIN unnest(gpt.attrs) ON TRUE"
- " WHERE gpt.relid = %u"
+ " LATERAL pg_get_publication_tables(p.pubname) gpt,"
+ " pg_class c"
+ " WHERE gpt.relid = %u AND c.oid = gpt.relid"
" AND p.pubname IN ( %s )",
lrel->remoteid,
pub_names.data);
@@ -812,27 +810,48 @@ fetch_remote_table_info(char *nspname, char *relname,
errmsg("could not fetch column list info for table \"%s.%s\" from publisher: %s",
nspname, relname, pubres->err)));
+ first = true;
+
/*
- * Merge the column lists (from different publications) by creating a
- * single bitmap with all the attnums. If we find a NULL value, that
- * means one of the publications has no column list for the table
- * we're syncing.
+ * Traverse the column lists from different publications and build a
+ * single bitmap with the attnums.
+ *
+ * During the loop, check that if all the column lists are the same and
+ * report an error if not.
+ *
+ * If we find a NULL value, that means one of the publications has no
+ * column list for the table we're syncing.
*/
slot = MakeSingleTupleTableSlot(pubres->tupledesc, &TTSOpsMinimalTuple);
while (tuplestore_gettupleslot(pubres->tuplestore, true, false, slot))
{
Datum cfval = slot_getattr(slot, 1, &isnull);
+ Bitmapset *cols = NULL;
- /* NULL means empty column list, so we're done. */
- if (isnull)
+ if (!isnull)
{
- bms_free(included_cols);
- included_cols = NULL;
- break;
+ ArrayType *arr;
+ int nelems;
+ int16 *elems;
+
+ arr = DatumGetArrayTypeP(cfval);
+ nelems = ARR_DIMS(arr)[0];
+ elems = (int16 *) ARR_DATA_PTR(arr);
+
+ for (int i = 0; i < nelems; i++)
+ cols = bms_add_member(cols, elems[i]);
}
- included_cols = bms_add_member(included_cols,
- DatumGetInt16(cfval));
+ if (first)
+ {
+ included_cols = cols;
+ first = false;
+ }
+ else if (!bms_equal(included_cols, cols))
+ ereport(ERROR,
+ errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot use different column lists for table \"%s.%s\" in different publications",
+ nspname, relname));
ExecClearTuple(slot);
}
diff --git a/src/backend/replication/pgoutput/pgoutput.c b/src/backend/replication/pgoutput/pgoutput.c
index 42c06af..4f58df2 100644
--- a/src/backend/replication/pgoutput/pgoutput.c
+++ b/src/backend/replication/pgoutput/pgoutput.c
@@ -979,12 +979,17 @@ pgoutput_column_list_init(PGOutputData *data, List *publications,
RelationSyncEntry *entry)
{
ListCell *lc;
+ bool first = true;
+ Relation relation = RelationIdGetRelation(entry->publish_as_relid);
/*
* Find if there are any column lists for this relation. If there are,
- * build a bitmap merging all the column lists.
+ * build a bitmap using the column lists.
*
- * All the given publication-table mappings must be checked.
+ * Note that we don't support the case where column list is different for
+ * the same table when combining publications. But we still need to check
+ * all the given publication-table mappings and report an error if any
+ * publications have different column list.
*
* Multiple publications might have multiple column lists for this
* relation.
@@ -997,12 +1002,7 @@ pgoutput_column_list_init(PGOutputData *data, List *publications,
Publication *pub = lfirst(lc);
HeapTuple cftuple = NULL;
Datum cfdatum = 0;
-
- /*
- * Assume there's no column list. Only if we find pg_publication_rel
- * entry with a column list we'll switch it to false.
- */
- bool pub_no_list = true;
+ Bitmapset *cols = NULL;
/*
* If the publication is FOR ALL TABLES then it is treated the same as
@@ -1011,6 +1011,8 @@ pgoutput_column_list_init(PGOutputData *data, List *publications,
*/
if (!pub->alltables)
{
+ bool pub_no_list = true;
+
/*
* Check for the presence of a column list in this publication.
*
@@ -1024,51 +1026,48 @@ pgoutput_column_list_init(PGOutputData *data, List *publications,
if (HeapTupleIsValid(cftuple))
{
- /*
- * Lookup the column list attribute.
- *
- * Note: We update the pub_no_list value directly, because if
- * the value is NULL, we have no list (and vice versa).
- */
+ /* Lookup the column list attribute. */
cfdatum = SysCacheGetAttr(PUBLICATIONRELMAP, cftuple,
Anum_pg_publication_rel_prattrs,
&pub_no_list);
- /*
- * Build the column list bitmap in the per-entry context.
- *
- * We need to merge column lists from all publications, so we
- * update the same bitmapset. If the column list is null, we
- * interpret it as replicating all columns.
- */
+ /* Build the column list bitmap in the per-entry context. */
if (!pub_no_list) /* when not null */
{
pgoutput_ensure_entry_cxt(data, entry);
- entry->columns = pub_collist_to_bitmapset(entry->columns,
- cfdatum,
- entry->entry_cxt);
+ cols = pub_collist_to_bitmapset(cols, cfdatum,
+ entry->entry_cxt);
+
+ /*
+ * If column list includes all the columns of the table,
+ * set it to NULL.
+ */
+ if (bms_num_members(cols) == RelationGetNumberOfAttributes(relation))
+ {
+ bms_free(cols);
+ cols = NULL;
+ }
}
+
+ ReleaseSysCache(cftuple);
}
}
- /*
- * Found a publication with no column list, so we're done. But first
- * discard column list we might have from preceding publications.
- */
- if (pub_no_list)
+ if (first)
{
- if (cftuple)
- ReleaseSysCache(cftuple);
-
- bms_free(entry->columns);
- entry->columns = NULL;
-
- break;
+ entry->columns = cols;
+ first = false;
}
-
- ReleaseSysCache(cftuple);
+ else if (!bms_equal(entry->columns, cols))
+ ereport(ERROR,
+ errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot use different column lists for table \"%s.%s\" in different publications",
+ get_namespace_name(RelationGetNamespace(relation)),
+ RelationGetRelationName(relation)));
} /* loop all subscribed publications */
+
+ RelationClose(relation);
}
/*
diff --git a/src/test/subscription/t/031_column_list.pl b/src/test/subscription/t/031_column_list.pl
index 19812e1..b454a05 100644
--- a/src/test/subscription/t/031_column_list.pl
+++ b/src/test/subscription/t/031_column_list.pl
@@ -21,6 +21,8 @@ $node_subscriber->start;
my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres';
+my $offset = 0;
+
sub wait_for_subscription_sync
{
my ($node) = @_;
@@ -361,13 +363,13 @@ is( $result, qq(1|abc
2|xyz), 'update on column tab2.c is not replicated');
-# TEST: add a table to two publications with different column lists, and
+# TEST: add a table to two publications with same column lists, and
# create a single subscription replicating both publications
$node_publisher->safe_psql(
'postgres', qq(
CREATE TABLE tab5 (a int PRIMARY KEY, b int, c int, d int);
CREATE PUBLICATION pub2 FOR TABLE tab5 (a, b);
- CREATE PUBLICATION pub3 FOR TABLE tab5 (a, d);
+ CREATE PUBLICATION pub3 FOR TABLE tab5 (a, b);
-- insert a couple initial records
INSERT INTO tab5 VALUES (1, 11, 111, 1111);
@@ -388,8 +390,7 @@ wait_for_subscription_sync($node_subscriber);
$node_publisher->wait_for_catchup('sub1');
-# insert data and make sure all the columns (union of the columns lists)
-# get fully replicated
+# insert data and make sure the columns in column list get fully replicated
$node_publisher->safe_psql(
'postgres', qq(
INSERT INTO tab5 VALUES (3, 33, 333, 3333);
@@ -399,42 +400,12 @@ $node_publisher->safe_psql(
$node_publisher->wait_for_catchup('sub1');
is( $node_subscriber->safe_psql('postgres', "SELECT * FROM tab5 ORDER BY a"),
- qq(1|11|1111
-2|22|2222
-3|33|3333
-4|44|4444),
+ qq(1|11|
+2|22|
+3|33|
+4|44|),
'overlapping publications with overlapping column lists');
-# and finally, remove the column list for one of the publications, which
-# means replicating all columns (removing the column list), but first add
-# the missing column to the table on subscriber
-$node_publisher->safe_psql(
- 'postgres', qq(
- ALTER PUBLICATION pub3 SET TABLE tab5;
-));
-
-$node_subscriber->safe_psql(
- 'postgres', qq(
- ALTER SUBSCRIPTION sub1 REFRESH PUBLICATION;
- ALTER TABLE tab5 ADD COLUMN c INT;
-));
-
-wait_for_subscription_sync($node_subscriber);
-
-$node_publisher->safe_psql(
- 'postgres', qq(
- INSERT INTO tab5 VALUES (5, 55, 555, 5555);
-));
-
-$node_publisher->wait_for_catchup('sub1');
-
-is( $node_subscriber->safe_psql('postgres', "SELECT * FROM tab5 ORDER BY a"),
- qq(1|11|1111|
-2|22|2222|
-3|33|3333|
-4|44|4444|
-5|55|5555|555),
- 'overlapping publications with overlapping column lists');
# TEST: create a table with a column list, then change the replica
# identity by replacing a primary key (but use a different column in
@@ -900,57 +871,21 @@ is( $node_subscriber->safe_psql(
3|),
'partitions with different replica identities not replicated correctly');
-# TEST: With a table included in multiple publications, we should use a
-# union of the column lists. So with column lists (a,b) and (a,c) we
-# should replicate (a,b,c).
-$node_publisher->safe_psql(
- 'postgres', qq(
- CREATE TABLE test_mix_1 (a int PRIMARY KEY, b int, c int);
- CREATE PUBLICATION pub_mix_1 FOR TABLE test_mix_1 (a, b);
- CREATE PUBLICATION pub_mix_2 FOR TABLE test_mix_1 (a, c);
-
- -- initial data
- INSERT INTO test_mix_1 VALUES (1, 2, 3);
-));
-
-$node_subscriber->safe_psql(
- 'postgres', qq(
- CREATE TABLE test_mix_1 (a int PRIMARY KEY, b int, c int);
- ALTER SUBSCRIPTION sub1 SET PUBLICATION pub_mix_1, pub_mix_2;
-));
-
-wait_for_subscription_sync($node_subscriber);
-
-$node_publisher->safe_psql(
- 'postgres', qq(
- INSERT INTO test_mix_1 VALUES (4, 5, 6);
-));
-
-$node_publisher->wait_for_catchup('sub1');
-
-is( $node_subscriber->safe_psql(
- 'postgres', "SELECT * FROM test_mix_1 ORDER BY a"),
- qq(1|2|3
-4|5|6),
- 'a mix of publications should use a union of column list');
-
-
-# TEST: With a table included in multiple publications, we should use a
-# union of the column lists. If any of the publications is FOR ALL
-# TABLES, we should replicate all columns.
+# TEST: With a table included in the publications is FOR ALL TABLES, it
+# means replicate all columns.
# drop unnecessary tables, so as not to interfere with the FOR ALL TABLES
$node_publisher->safe_psql(
'postgres', qq(
- DROP TABLE tab1, tab2, tab3, tab4, tab5, tab6, tab7, test_mix_1,
+ DROP TABLE tab1, tab2, tab3, tab4, tab5, tab6, tab7,
test_part, test_part_a, test_part_b, test_part_c, test_part_d;
));
$node_publisher->safe_psql(
'postgres', qq(
CREATE TABLE test_mix_2 (a int PRIMARY KEY, b int, c int);
- CREATE PUBLICATION pub_mix_3 FOR TABLE test_mix_2 (a, b);
+ CREATE PUBLICATION pub_mix_3 FOR TABLE test_mix_2 (a, b, c);
CREATE PUBLICATION pub_mix_4 FOR ALL TABLES;
-- initial data
@@ -976,12 +911,11 @@ $node_publisher->wait_for_catchup('sub1');
is( $node_subscriber->safe_psql('postgres', "SELECT * FROM test_mix_2"),
qq(1|2|3
4|5|6),
- 'a mix of publications should use a union of column list');
+ 'all columns should be replicated');
-# TEST: With a table included in multiple publications, we should use a
-# union of the column lists. If any of the publications is FOR ALL
-# TABLES IN SCHEMA, we should replicate all columns.
+# TEST: With a table included in the publication which is FOR ALL TABLES
+# IN SCHEMA, it means replicate all columns.
$node_subscriber->safe_psql(
'postgres', qq(
@@ -993,7 +927,7 @@ $node_publisher->safe_psql(
'postgres', qq(
DROP TABLE test_mix_2;
CREATE TABLE test_mix_3 (a int PRIMARY KEY, b int, c int);
- CREATE PUBLICATION pub_mix_5 FOR TABLE test_mix_3 (a, b);
+ CREATE PUBLICATION pub_mix_5 FOR TABLE test_mix_3 (a, b, c);
CREATE PUBLICATION pub_mix_6 FOR ALL TABLES IN SCHEMA public;
-- initial data
@@ -1017,7 +951,7 @@ $node_publisher->wait_for_catchup('sub1');
is( $node_subscriber->safe_psql('postgres', "SELECT * FROM test_mix_3"),
qq(1|2|3
4|5|6),
- 'a mix of publications should use a union of column list');
+ 'all columns should be replicated');
# TEST: Check handling of publish_via_partition_root - if a partition is
@@ -1074,7 +1008,7 @@ is( $node_subscriber->safe_psql(
# TEST: Multiple publications which publish schema of parent table and
# partition. The partition is published through two publications, once
# through a schema (so no column list) containing the parent, and then
-# also directly (with a columns list). The expected outcome is there is
+# also directly (with all columns). The expected outcome is there is
# no column list.
$node_publisher->safe_psql(
@@ -1086,7 +1020,7 @@ $node_publisher->safe_psql(
CREATE TABLE t_1 PARTITION OF s1.t FOR VALUES FROM (1) TO (10);
CREATE PUBLICATION pub1 FOR ALL TABLES IN SCHEMA s1;
- CREATE PUBLICATION pub2 FOR TABLE t_1(b);
+ CREATE PUBLICATION pub2 FOR TABLE t_1(a, b, c);
-- initial data
INSERT INTO s1.t VALUES (1, 2, 3);
@@ -1233,6 +1167,27 @@ is( $node_subscriber->safe_psql(
'publication containing both parent and child relation');
+# TEST: With a table included in multiple publications with different column
+# lists, we should catch the error in the log
+
+$node_publisher->safe_psql('postgres', qq(
+ CREATE TABLE test_mix_1 (a int PRIMARY KEY, b int, c int);
+ CREATE PUBLICATION pub_mix_1 FOR TABLE test_mix_1 (a, b);
+ CREATE PUBLICATION pub_mix_2 FOR TABLE test_mix_1 (a, c);
+));
+
+$node_subscriber->safe_psql('postgres', qq(
+ CREATE TABLE test_mix_1 (a int PRIMARY KEY, b int, c int);
+ ALTER SUBSCRIPTION sub1 SET PUBLICATION pub_mix_1, pub_mix_2 WITH (copy_data = false);
+));
+
+$node_publisher->wait_for_catchup('sub1');
+
+my $logfile = slurp_file($node_subscriber->logfile, $offset);
+ok( $logfile =~
+ qr/cannot use different column lists for table "public.test_mix_1" in different publications/,
+ 'different column lists detected');
+
$node_subscriber->stop('fast');
$node_publisher->stop('fast');
--
2.7.2.windows.1
[application/octet-stream] 0001-extent-pg_publication_tables.patch (12.6K, ../../OS0PR01MB571698DEFCE9ACC6DC40510794CA9@OS0PR01MB5716.jpnprd01.prod.outlook.com/3-0001-extent-pg_publication_tables.patch)
download | inline diff:
From 16d36b556d066480cda4dd170e13183aaf9d8173 Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Fri, 13 May 2022 11:35:14 +0800
Subject: [PATCH] extent pg_publication_tables
Extend the pg_publication_tables view and pg_get_publication_tables function
so that they can return the column list and row fitler of the table. It will
make it easier for users and developers to fetch the column lists and row filters.
---
doc/src/sgml/catalogs.sgml | 20 +++++++++++
src/backend/catalog/pg_publication.c | 52 ++++++++++++++++++++++++++++-
src/backend/catalog/system_views.sql | 4 ++-
src/backend/replication/logical/tablesync.c | 14 +++-----
src/include/catalog/pg_proc.dat | 6 ++--
src/test/regress/expected/publication.out | 42 +++++++++++------------
src/test/regress/expected/rules.out | 6 ++--
7 files changed, 106 insertions(+), 38 deletions(-)
diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index a533a21..fd0c61b 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -11687,6 +11687,26 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
Name of table
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>columnlist</structfield> <type>int2vector</type>
+ (references <link linkend="catalog-pg-publication-rel"><structname>pg_publication_rel</structname></link>.<structfield>prattrs</structfield>)
+ </para>
+ <para>
+ Column list of table
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>rowfilter</structfield> <type>text</type>
+ (references <link linkend="catalog-pg-publication-rel"><structname>pg_publication_rel</structname></link>.<structfield>prqual</structfield>)
+ </para>
+ <para>
+ Row filter of table
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index e2c8bcb..3fe607e 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -1082,6 +1082,7 @@ get_publication_name(Oid pubid, bool missing_ok)
Datum
pg_get_publication_tables(PG_FUNCTION_ARGS)
{
+#define NUM_PUBLICATOIN_TABLES_ELEM 3
FuncCallContext *funcctx;
char *pubname = text_to_cstring(PG_GETARG_TEXT_PP(0));
Publication *publication;
@@ -1090,6 +1091,7 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
/* stuff done only on the first call of the function */
if (SRF_IS_FIRSTCALL())
{
+ TupleDesc tupdesc;
MemoryContext oldcontext;
/* create a function context for cross-call persistence */
@@ -1136,6 +1138,16 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
tables = filter_partitions(tables);
}
+ /* Construct a tuple descriptor for the result rows. */
+ tupdesc = CreateTemplateTupleDesc(NUM_PUBLICATOIN_TABLES_ELEM);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 1, "relid",
+ OIDOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 2, "attrs",
+ INT2VECTOROID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 3, "qual",
+ PG_NODE_TREEOID, -1, 0);
+
+ funcctx->tuple_desc = BlessTupleDesc(tupdesc);
funcctx->user_fctx = (void *) tables;
MemoryContextSwitchTo(oldcontext);
@@ -1147,9 +1159,47 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
if (funcctx->call_cntr < list_length(tables))
{
+ HeapTuple pubtuple = NULL;
+ HeapTuple rettuple;
Oid relid = list_nth_oid(tables, funcctx->call_cntr);
+ Datum values[NUM_PUBLICATOIN_TABLES_ELEM];
+ bool nulls[NUM_PUBLICATOIN_TABLES_ELEM];
+
+ /*
+ * Form tuple with appropriate data.
+ */
+ MemSet(nulls, 0, sizeof(nulls));
+ MemSet(values, 0, sizeof(values));
+
+ publication = GetPublicationByName(pubname, false);
+
+ values[0] = ObjectIdGetDatum(relid);
+
+ pubtuple = SearchSysCacheCopy2(PUBLICATIONRELMAP,
+ ObjectIdGetDatum(relid),
+ ObjectIdGetDatum(publication->oid));
+
+ if (HeapTupleIsValid(pubtuple))
+ {
+ /* Lookup the column list attribute. */
+ values[1] = SysCacheGetAttr(PUBLICATIONRELMAP, pubtuple,
+ Anum_pg_publication_rel_prattrs,
+ &(nulls[1]));
+
+ /* Null indicates no filter. */
+ values[2] = SysCacheGetAttr(PUBLICATIONRELMAP, pubtuple,
+ Anum_pg_publication_rel_prqual,
+ &(nulls[2]));
+ }
+ else
+ {
+ nulls[1] = true;
+ nulls[2] = true;
+ }
+
+ rettuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
- SRF_RETURN_NEXT(funcctx, ObjectIdGetDatum(relid));
+ SRF_RETURN_NEXT(funcctx, HeapTupleGetDatum(rettuple));
}
SRF_RETURN_DONE(funcctx);
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 0fc614e..f758f15 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -368,7 +368,9 @@ CREATE VIEW pg_publication_tables AS
SELECT
P.pubname AS pubname,
N.nspname AS schemaname,
- C.relname AS tablename
+ C.relname AS tablename,
+ GPT.attrs AS columnlist,
+ pg_get_expr(GPT.qual, GPT.relid) AS rowfilter
FROM pg_publication P,
LATERAL pg_get_publication_tables(P.pubname) GPT,
pg_class C JOIN pg_namespace N ON (N.oid = C.relnamespace)
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index b03e0f5..994c7a0 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -795,15 +795,12 @@ fetch_remote_table_info(char *nspname, char *relname,
resetStringInfo(&cmd);
appendStringInfo(&cmd,
"SELECT DISTINCT unnest"
- " FROM pg_publication p"
- " LEFT OUTER JOIN pg_publication_rel pr"
- " ON (p.oid = pr.prpubid AND pr.prrelid = %u)"
- " LEFT OUTER JOIN unnest(pr.prattrs) ON TRUE,"
+ " FROM pg_publication p,"
" LATERAL pg_get_publication_tables(p.pubname) gpt"
+ " LEFT OUTER JOIN unnest(gpt.attrs) ON TRUE"
" WHERE gpt.relid = %u"
" AND p.pubname IN ( %s )",
lrel->remoteid,
- lrel->remoteid,
pub_names.data);
pubres = walrcv_exec(LogRepWorkerWalRcvConn, cmd.data,
@@ -965,15 +962,12 @@ fetch_remote_table_info(char *nspname, char *relname,
/* Check for row filters. */
resetStringInfo(&cmd);
appendStringInfo(&cmd,
- "SELECT DISTINCT pg_get_expr(pr.prqual, pr.prrelid)"
- " FROM pg_publication p"
- " LEFT OUTER JOIN pg_publication_rel pr"
- " ON (p.oid = pr.prpubid AND pr.prrelid = %u),"
+ "SELECT DISTINCT pg_get_expr(gpt.qual, gpt.relid)"
+ " FROM pg_publication p,"
" LATERAL pg_get_publication_tables(p.pubname) gpt"
" WHERE gpt.relid = %u"
" AND p.pubname IN ( %s )",
lrel->remoteid,
- lrel->remoteid,
pub_names.data);
res = walrcv_exec(LogRepWorkerWalRcvConn, cmd.data, 1, qualRow);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index babe16f..cf6235d 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -11675,9 +11675,9 @@
# publications
{ oid => '6119', descr => 'get OIDs of tables in a publication',
proname => 'pg_get_publication_tables', prorows => '1000', proretset => 't',
- provolatile => 's', prorettype => 'oid', proargtypes => 'text',
- proallargtypes => '{text,oid}', proargmodes => '{i,o}',
- proargnames => '{pubname,relid}', prosrc => 'pg_get_publication_tables' },
+ provolatile => 's', prorettype => 'record', proargtypes => 'text',
+ proallargtypes => '{text,oid,int2vector,pg_node_tree}', proargmodes => '{i,o,o,o}',
+ proargnames => '{pubname,relid,attrs,qual}', prosrc => 'pg_get_publication_tables' },
{ oid => '6121',
descr => 'returns whether a relation can be part of a publication',
proname => 'pg_relation_is_publishable', provolatile => 's',
diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out
index 398c0f3..4ebafb3 100644
--- a/src/test/regress/expected/publication.out
+++ b/src/test/regress/expected/publication.out
@@ -1585,52 +1585,52 @@ CREATE TABLE sch2.tbl1_part1 PARTITION OF sch1.tbl1 FOR VALUES FROM (1) to (10);
-- Schema publication that does not include the schema that has the parent table
CREATE PUBLICATION pub FOR ALL TABLES IN SCHEMA sch2 WITH (PUBLISH_VIA_PARTITION_ROOT=1);
SELECT * FROM pg_publication_tables;
- pubname | schemaname | tablename
----------+------------+------------
- pub | sch2 | tbl1_part1
+ pubname | schemaname | tablename | columnlist | rowfilter
+---------+------------+------------+------------+-----------
+ pub | sch2 | tbl1_part1 | |
(1 row)
DROP PUBLICATION pub;
-- Table publication that does not include the parent table
CREATE PUBLICATION pub FOR TABLE sch2.tbl1_part1 WITH (PUBLISH_VIA_PARTITION_ROOT=1);
SELECT * FROM pg_publication_tables;
- pubname | schemaname | tablename
----------+------------+------------
- pub | sch2 | tbl1_part1
+ pubname | schemaname | tablename | columnlist | rowfilter
+---------+------------+------------+------------+-----------
+ pub | sch2 | tbl1_part1 | |
(1 row)
-- Table publication that includes both the parent table and the child table
ALTER PUBLICATION pub ADD TABLE sch1.tbl1;
SELECT * FROM pg_publication_tables;
- pubname | schemaname | tablename
----------+------------+-----------
- pub | sch1 | tbl1
+ pubname | schemaname | tablename | columnlist | rowfilter
+---------+------------+-----------+------------+-----------
+ pub | sch1 | tbl1 | |
(1 row)
DROP PUBLICATION pub;
-- Schema publication that does not include the schema that has the parent table
CREATE PUBLICATION pub FOR ALL TABLES IN SCHEMA sch2 WITH (PUBLISH_VIA_PARTITION_ROOT=0);
SELECT * FROM pg_publication_tables;
- pubname | schemaname | tablename
----------+------------+------------
- pub | sch2 | tbl1_part1
+ pubname | schemaname | tablename | columnlist | rowfilter
+---------+------------+------------+------------+-----------
+ pub | sch2 | tbl1_part1 | |
(1 row)
DROP PUBLICATION pub;
-- Table publication that does not include the parent table
CREATE PUBLICATION pub FOR TABLE sch2.tbl1_part1 WITH (PUBLISH_VIA_PARTITION_ROOT=0);
SELECT * FROM pg_publication_tables;
- pubname | schemaname | tablename
----------+------------+------------
- pub | sch2 | tbl1_part1
+ pubname | schemaname | tablename | columnlist | rowfilter
+---------+------------+------------+------------+-----------
+ pub | sch2 | tbl1_part1 | |
(1 row)
-- Table publication that includes both the parent table and the child table
ALTER PUBLICATION pub ADD TABLE sch1.tbl1;
SELECT * FROM pg_publication_tables;
- pubname | schemaname | tablename
----------+------------+------------
- pub | sch2 | tbl1_part1
+ pubname | schemaname | tablename | columnlist | rowfilter
+---------+------------+------------+------------+-----------
+ pub | sch2 | tbl1_part1 | |
(1 row)
DROP PUBLICATION pub;
@@ -1643,9 +1643,9 @@ CREATE TABLE sch1.tbl1_part3 (a int) PARTITION BY RANGE(a);
ALTER TABLE sch1.tbl1 ATTACH PARTITION sch1.tbl1_part3 FOR VALUES FROM (20) to (30);
CREATE PUBLICATION pub FOR ALL TABLES IN SCHEMA sch1 WITH (PUBLISH_VIA_PARTITION_ROOT=1);
SELECT * FROM pg_publication_tables;
- pubname | schemaname | tablename
----------+------------+-----------
- pub | sch1 | tbl1
+ pubname | schemaname | tablename | columnlist | rowfilter
+---------+------------+-----------+------------+-----------
+ pub | sch1 | tbl1 | |
(1 row)
RESET client_min_messages;
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 21effe8..c7f8300 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1437,9 +1437,11 @@ pg_prepared_xacts| SELECT p.transaction,
LEFT JOIN pg_database d ON ((p.dbid = d.oid)));
pg_publication_tables| SELECT p.pubname,
n.nspname AS schemaname,
- c.relname AS tablename
+ c.relname AS tablename,
+ gpt.attrs AS columnlist,
+ pg_get_expr(gpt.qual, gpt.relid) AS rowfilter
FROM pg_publication p,
- LATERAL pg_get_publication_tables((p.pubname)::text) gpt(relid),
+ LATERAL pg_get_publication_tables((p.pubname)::text) gpt(relid, attrs, qual),
(pg_class c
JOIN pg_namespace n ON ((n.oid = c.relnamespace)))
WHERE (c.oid = gpt.relid);
--
2.7.2.windows.1
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-16 06:10 Amit Kapila <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 2 replies; 58+ messages in thread
From: Amit Kapila @ 2022-05-16 06:10 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Tomas Vondra <[email protected]>; Alvaro Herrera <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On Fri, May 13, 2022 at 11:32 AM [email protected]
<[email protected]> wrote:
>
> On Thursday, May 12, 2022 2:45 PM Amit Kapila <[email protected]> wrote:
> >
> > On Wed, May 11, 2022 at 12:55 PM [email protected]
> > <[email protected]> wrote:
> >
> > Few comments:
> > ===============
> > 1.
> > initStringInfo(&cmd);
> > - appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname,
> > t.tablename\n"
> > - " FROM pg_catalog.pg_publication_tables t\n"
> > + appendStringInfoString(&cmd,
> > + "SELECT DISTINCT t.schemaname,\n"
> > + " t.tablename,\n"
> > + " (CASE WHEN (array_length(pr.prattrs, 1) = t.relnatts)\n"
> > + " THEN NULL ELSE pr.prattrs END)\n"
> > + " FROM (SELECT P.pubname AS pubname,\n"
> > + " N.nspname AS schemaname,\n"
> > + " C.relname AS tablename,\n"
> > + " P.oid AS pubid,\n"
> > + " C.oid AS reloid,\n"
> > + " C.relnatts\n"
> > + " FROM pg_publication P,\n"
> > + " LATERAL pg_get_publication_tables(P.pubname) GPT,\n"
> > + " pg_class C JOIN pg_namespace N\n"
> > + " ON (N.oid = C.relnamespace)\n"
> > + " WHERE C.oid = GPT.relid) t\n"
> > + " LEFT OUTER JOIN pg_publication_rel pr\n"
> > + " ON (t.pubid = pr.prpubid AND\n"
> > + " pr.prrelid = reloid)\n"
> >
> > Can we modify pg_publication_tables to get the row filter and column list and
> > then use it directly instead of constructing this query?
>
> Agreed. If we can get columnlist and rowfilter from pg_publication_tables, it
> will be more convenient. And I think users that want to fetch the columnlist
> and rowfilter of table can also benefit from it.
>
After the change for this, we will give an error on combining
publications where one of the publications specifies all columns in
the table and the other doesn't provide any columns. We should not
give an error as both mean all columns.
>
> Attach the new version patch which addressed these comments and update the
> document. 0001 patch is to extent the view and 0002 patch is to add restriction
> for column list.
>
Few comments:
=================
1.
postgres=# select * from pg_publication_tables;
pubname | schemaname | tablename | columnlist | rowfilter
---------+------------+-----------+------------+-----------
pub1 | public | t1 | |
pub2 | public | t1 | 1 2 | (c3 < 10)
(2 rows)
I think it is better to display column names for columnlist in the
exposed view similar to attnames in the pg_stats_ext view. I think
that will make it easier for users to understand this information.
2.
{ oid => '6119', descr => 'get OIDs of tables in a publication',
proname => 'pg_get_publication_tables', prorows => '1000', proretset => 't',
- provolatile => 's', prorettype => 'oid', proargtypes => 'text',
- proallargtypes => '{text,oid}', proargmodes => '{i,o}',
- proargnames => '{pubname,relid}', prosrc => 'pg_get_publication_tables' },
+ provolatile => 's', prorettype => 'record', proargtypes => 'text',
+ proallargtypes => '{text,oid,int2vector,pg_node_tree}', proargmodes
=> '{i,o,o,o}',
I think we should change the "descr" to something like: 'get
information of tables in a publication'
3.
+
+ /*
+ * We only throw a warning here so that the subcription can still be
+ * created and let user aware that something is going to fail later and
+ * they can fix the publications afterwards.
+ */
+ if (list_member(tablelist, rv))
+ ereport(WARNING,
+ errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot use different column lists for table \"%s.%s\" in
different publications",
+ nspname, relname));
Can we extend this comment to explain the case where after Alter
Publication, if the user dumps and restores back the subscription,
there is a possibility that "CREATE SUBSCRIPTION" won't work if we
give ERROR here instead of WARNING?
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 58+ messages in thread
* RE: bogus: logical replication rows/cols combinations
@ 2022-05-16 12:34 [email protected] <[email protected]>
parent: Amit Kapila <[email protected]>
1 sibling, 3 replies; 58+ messages in thread
From: [email protected] @ 2022-05-16 12:34 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: Tomas Vondra <[email protected]>; Alvaro Herrera <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On Monday, May 16, 2022 2:10 PM Amit Kapila <[email protected]>
>
> On Fri, May 13, 2022 at 11:32 AM [email protected]
> <[email protected]> wrote:
> >
> > On Thursday, May 12, 2022 2:45 PM Amit Kapila <[email protected]>
> wrote:
> > >
> > > On Wed, May 11, 2022 at 12:55 PM [email protected]
> > > <[email protected]> wrote:
> > >
> > > Few comments:
> > > ===============
> > > 1.
> > > initStringInfo(&cmd);
> > > - appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname,
> > > t.tablename\n"
> > > - " FROM pg_catalog.pg_publication_tables t\n"
> > > + appendStringInfoString(&cmd,
> > > + "SELECT DISTINCT t.schemaname,\n"
> > > + " t.tablename,\n"
> > > + " (CASE WHEN (array_length(pr.prattrs, 1) =
> t.relnatts)\n"
> > > + " THEN NULL ELSE pr.prattrs END)\n"
> > > + " FROM (SELECT P.pubname AS pubname,\n"
> > > + " N.nspname AS schemaname,\n"
> > > + " C.relname AS tablename,\n"
> > > + " P.oid AS pubid,\n"
> > > + " C.oid AS reloid,\n"
> > > + " C.relnatts\n"
> > > + " FROM pg_publication P,\n"
> > > + " LATERAL pg_get_publication_tables(P.pubname) GPT,\n"
> > > + " pg_class C JOIN pg_namespace N\n"
> > > + " ON (N.oid = C.relnamespace)\n"
> > > + " WHERE C.oid = GPT.relid) t\n"
> > > + " LEFT OUTER JOIN pg_publication_rel pr\n"
> > > + " ON (t.pubid = pr.prpubid AND\n"
> > > + " pr.prrelid = reloid)\n"
> > >
> > > Can we modify pg_publication_tables to get the row filter and column list
> and
> > > then use it directly instead of constructing this query?
> >
> > Agreed. If we can get columnlist and rowfilter from pg_publication_tables, it
> > will be more convenient. And I think users that want to fetch the columnlist
> > and rowfilter of table can also benefit from it.
> >
>
> After the change for this, we will give an error on combining
> publications where one of the publications specifies all columns in
> the table and the other doesn't provide any columns. We should not
> give an error as both mean all columns.
Thanks for the comments. Fixed.
> >
> > Attach the new version patch which addressed these comments and update
> the
> > document. 0001 patch is to extent the view and 0002 patch is to add
> restriction
> > for column list.
> >
>
> Few comments:
> =================
> 1.
> postgres=# select * from pg_publication_tables;
> pubname | schemaname | tablename | columnlist | rowfilter
> ---------+------------+-----------+------------+-----------
> pub1 | public | t1 | |
> pub2 | public | t1 | 1 2 | (c3 < 10)
> (2 rows)
>
> I think it is better to display column names for columnlist in the
> exposed view similar to attnames in the pg_stats_ext view. I think
> that will make it easier for users to understand this information.
Agreed and changed.
> 2.
> { oid => '6119', descr => 'get OIDs of tables in a publication',
> proname => 'pg_get_publication_tables', prorows => '1000', proretset =>
> 't',
> - provolatile => 's', prorettype => 'oid', proargtypes => 'text',
> - proallargtypes => '{text,oid}', proargmodes => '{i,o}',
> - proargnames => '{pubname,relid}', prosrc => 'pg_get_publication_tables' },
> + provolatile => 's', prorettype => 'record', proargtypes => 'text',
> + proallargtypes => '{text,oid,int2vector,pg_node_tree}', proargmodes
> => '{i,o,o,o}',
>
> I think we should change the "descr" to something like: 'get
> information of tables in a publication'
Changed.
> 3.
> +
> + /*
> + * We only throw a warning here so that the subcription can still be
> + * created and let user aware that something is going to fail later and
> + * they can fix the publications afterwards.
> + */
> + if (list_member(tablelist, rv))
> + ereport(WARNING,
> + errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
> + errmsg("cannot use different column lists for table \"%s.%s\" in
> different publications",
> + nspname, relname));
>
> Can we extend this comment to explain the case where after Alter
> Publication, if the user dumps and restores back the subscription,
> there is a possibility that "CREATE SUBSCRIPTION" won't work if we
> give ERROR here instead of WARNING?
After rethinking about this, It seems ok to report an ERROR here as the pg_dump
of subscription always set (connect = false). So, we won't hit the check when
restore the dump which means the restore can be successful even if user change
the publication afterwards. Based on this, I have changed the warning to error.
Attach the new version patch.
Best regards,
Hou zj
Attachments:
[application/octet-stream] v2-0002-Prohibit-combining-publications-with-different-colum.patch (19.5K, ../../OS0PR01MB5716D2CF3F2E39A6246A76CE94CF9@OS0PR01MB5716.jpnprd01.prod.outlook.com/2-v2-0002-Prohibit-combining-publications-with-different-colum.patch)
download | inline diff:
From aa3fa3a3de41b37f292308654c697a8c23ebb6a7 Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Fri, 13 May 2022 13:19:49 +0800
Subject: [PATCH] Prohibit combining publications with different column lists.
The main purpose of introducing a column list is to have statically
different shapes on publisher and subscriber or hide sensitive column
data. In both cases, it doesn't seems to make sense to combine column
lists.
So, we disallow the cases where the column list is different for the same
table when combining publications. It can be later extended to combine the
column lists for selective cases where required.
Reported-by: Alvaro Herrera
Author: Hou Zhijie
Reviewed-by: Amit Kapila
Discussion: https://postgr.es/m/[email protected]
---
doc/src/sgml/ref/alter_publication.sgml | 12 +++-
doc/src/sgml/ref/create_subscription.sgml | 5 ++
src/backend/commands/subscriptioncmds.c | 23 ++++--
src/backend/replication/logical/tablesync.c | 60 ++++++++++------
src/backend/replication/pgoutput/pgoutput.c | 77 ++++++++++----------
src/test/subscription/t/031_column_list.pl | 105 +++++-----------------------
6 files changed, 127 insertions(+), 155 deletions(-)
diff --git a/doc/src/sgml/ref/alter_publication.sgml b/doc/src/sgml/ref/alter_publication.sgml
index e2cce49..f03933a 100644
--- a/doc/src/sgml/ref/alter_publication.sgml
+++ b/doc/src/sgml/ref/alter_publication.sgml
@@ -116,7 +116,17 @@ ALTER PUBLICATION <replaceable class="parameter">name</replaceable> RENAME TO <r
<para>
Optionally, a column list can be specified. See <xref
- linkend="sql-createpublication"/> for details.
+ linkend="sql-createpublication"/> for details. Note that a subscription
+ having several publications in which the same table has been published
+ with different column lists is not supported. So, changing the column
+ lists of the tables being subscribed could cause inconsistency of column
+ lists among publications in which case <command>ALTER PUBLICATION</command>
+ command will be successful but later the WalSender in publisher or the
+ subscriber may throw an error. In this scenario, the user needs to
+ recreate the subscription after adjusting the column list or drop the
+ problematic publication using
+ <literal>ALTER SUBSCRIPTION ... DROP PUBLICATION</literal> and then add
+ it back after adjusting the column list.
</para>
<para>
diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml
index 203bb41..f6f82a0 100644
--- a/doc/src/sgml/ref/create_subscription.sgml
+++ b/doc/src/sgml/ref/create_subscription.sgml
@@ -356,6 +356,11 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl
</para>
<para>
+ Subscription having several publications in which the same table has been
+ published with different column lists is not supported.
+ </para>
+
+ <para>
We allow non-existent publications to be specified so that users can add
those later. This means
<link linkend="catalog-pg-subscription"><structname>pg_subscription</structname></link>
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 690cdaa..991b2c1 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1753,7 +1753,8 @@ AlterSubscriptionOwner_oid(Oid subid, Oid newOwnerId)
/*
* Get the list of tables which belong to specified publications on the
- * publisher connection.
+ * publisher connection. Also get the column list for each table and check if
+ * column lists are the same in different publications.
*/
static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
@@ -1761,17 +1762,18 @@ fetch_table_list(WalReceiverConn *wrconn, List *publications)
WalRcvExecResult *res;
StringInfoData cmd;
TupleTableSlot *slot;
- Oid tableRow[2] = {TEXTOID, TEXTOID};
+ Oid tableRow[3] = {TEXTOID, TEXTOID, NAMEARRAYOID};
List *tablelist = NIL;
initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename\n"
+ appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename, \n"
+ " t.attnames\n"
" FROM pg_catalog.pg_publication_tables t\n"
" WHERE t.pubname IN (");
get_publications_str(publications, &cmd, true);
appendStringInfoChar(&cmd, ')');
- res = walrcv_exec(wrconn, cmd.data, 2, tableRow);
+ res = walrcv_exec(wrconn, cmd.data, 3, tableRow);
pfree(cmd.data);
if (res->status != WALRCV_OK_TUPLES)
@@ -1795,7 +1797,18 @@ fetch_table_list(WalReceiverConn *wrconn, List *publications)
Assert(!isnull);
rv = makeRangeVar(nspname, relname, -1);
- tablelist = lappend(tablelist, rv);
+
+ /*
+ * We don't support the case where column list is different for the
+ * same table in different publications.
+ */
+ if (list_member(tablelist, rv))
+ ereport(ERROR,
+ errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot use different column lists for table \"%s.%s\" in different publications",
+ nspname, relname));
+ else
+ tablelist = lappend(tablelist, rv);
ExecClearTuple(slot);
}
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 994c7a0..9ce4b9f 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -771,7 +771,7 @@ fetch_remote_table_info(char *nspname, char *relname,
{
WalRcvExecResult *pubres;
TupleTableSlot *slot;
- Oid attrsRow[] = {INT2OID};
+ Oid attrsRow[] = {INT2VECTOROID};
StringInfoData pub_names;
bool first = true;
@@ -786,19 +786,17 @@ fetch_remote_table_info(char *nspname, char *relname,
/*
* Fetch info about column lists for the relation (from all the
- * publications). We unnest the int2vector values, because that makes
- * it easier to combine lists by simply adding the attnums to a new
- * bitmap (without having to parse the int2vector data). This
- * preserves NULL values, so that if one of the publications has no
- * column list, we'll know that.
+ * publications).
*/
resetStringInfo(&cmd);
appendStringInfo(&cmd,
- "SELECT DISTINCT unnest"
+ "SELECT DISTINCT"
+ " (CASE WHEN (array_length(gpt.attrs, 1) = c.relnatts)"
+ " THEN NULL ELSE gpt.attrs END)"
" FROM pg_publication p,"
- " LATERAL pg_get_publication_tables(p.pubname) gpt"
- " LEFT OUTER JOIN unnest(gpt.attrs) ON TRUE"
- " WHERE gpt.relid = %u"
+ " LATERAL pg_get_publication_tables(p.pubname) gpt,"
+ " pg_class c"
+ " WHERE gpt.relid = %u AND c.oid = gpt.relid"
" AND p.pubname IN ( %s )",
lrel->remoteid,
pub_names.data);
@@ -813,26 +811,42 @@ fetch_remote_table_info(char *nspname, char *relname,
nspname, relname, pubres->err)));
/*
- * Merge the column lists (from different publications) by creating a
- * single bitmap with all the attnums. If we find a NULL value, that
- * means one of the publications has no column list for the table
- * we're syncing.
+ * We don't support the case where column list is different for the
+ * same table when combining publications. So there should be only one
+ * row returned. Although we already checked this when creating
+ * subscription, we still need to check here in case the column list
+ * was changed afterwards.
+ */
+ if (tuplestore_tuple_count(pubres->tuplestore) > 1)
+ ereport(ERROR,
+ errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot use different column lists for table \"%s.%s\" in different publications",
+ nspname, relname));
+
+ /*
+ * Get the column list and build a single bitmap with the attnums.
+ *
+ * If we find a NULL value, it means all the columns should be
+ * replicated.
*/
slot = MakeSingleTupleTableSlot(pubres->tupledesc, &TTSOpsMinimalTuple);
- while (tuplestore_gettupleslot(pubres->tuplestore, true, false, slot))
+ if (tuplestore_gettupleslot(pubres->tuplestore, true, false, slot))
{
Datum cfval = slot_getattr(slot, 1, &isnull);
- /* NULL means empty column list, so we're done. */
- if (isnull)
+ if (!isnull)
{
- bms_free(included_cols);
- included_cols = NULL;
- break;
- }
+ ArrayType *arr;
+ int nelems;
+ int16 *elems;
+
+ arr = DatumGetArrayTypeP(cfval);
+ nelems = ARR_DIMS(arr)[0];
+ elems = (int16 *) ARR_DATA_PTR(arr);
- included_cols = bms_add_member(included_cols,
- DatumGetInt16(cfval));
+ for (natt = 0; natt < nelems; natt++)
+ included_cols = bms_add_member(included_cols, elems[natt]);
+ }
ExecClearTuple(slot);
}
diff --git a/src/backend/replication/pgoutput/pgoutput.c b/src/backend/replication/pgoutput/pgoutput.c
index 42c06af..3d888a5 100644
--- a/src/backend/replication/pgoutput/pgoutput.c
+++ b/src/backend/replication/pgoutput/pgoutput.c
@@ -979,30 +979,30 @@ pgoutput_column_list_init(PGOutputData *data, List *publications,
RelationSyncEntry *entry)
{
ListCell *lc;
+ bool first = true;
+ Relation relation = RelationIdGetRelation(entry->publish_as_relid);
/*
* Find if there are any column lists for this relation. If there are,
- * build a bitmap merging all the column lists.
+ * build a bitmap using the column lists.
*
- * All the given publication-table mappings must be checked.
+ * Note that we don't support the case where column list is different for
+ * the same table when combining publications. But we still need to check
+ * all the given publication-table mappings and report an error if any
+ * publications have different column list.
*
* Multiple publications might have multiple column lists for this
* relation.
*
* FOR ALL TABLES and FOR ALL TABLES IN SCHEMA implies "don't use column
- * list" so it takes precedence.
+ * list".
*/
foreach(lc, publications)
{
Publication *pub = lfirst(lc);
HeapTuple cftuple = NULL;
Datum cfdatum = 0;
-
- /*
- * Assume there's no column list. Only if we find pg_publication_rel
- * entry with a column list we'll switch it to false.
- */
- bool pub_no_list = true;
+ Bitmapset *cols = NULL;
/*
* If the publication is FOR ALL TABLES then it is treated the same as
@@ -1011,6 +1011,8 @@ pgoutput_column_list_init(PGOutputData *data, List *publications,
*/
if (!pub->alltables)
{
+ bool pub_no_list = true;
+
/*
* Check for the presence of a column list in this publication.
*
@@ -1024,51 +1026,48 @@ pgoutput_column_list_init(PGOutputData *data, List *publications,
if (HeapTupleIsValid(cftuple))
{
- /*
- * Lookup the column list attribute.
- *
- * Note: We update the pub_no_list value directly, because if
- * the value is NULL, we have no list (and vice versa).
- */
+ /* Lookup the column list attribute. */
cfdatum = SysCacheGetAttr(PUBLICATIONRELMAP, cftuple,
Anum_pg_publication_rel_prattrs,
&pub_no_list);
- /*
- * Build the column list bitmap in the per-entry context.
- *
- * We need to merge column lists from all publications, so we
- * update the same bitmapset. If the column list is null, we
- * interpret it as replicating all columns.
- */
+ /* Build the column list bitmap in the per-entry context. */
if (!pub_no_list) /* when not null */
{
pgoutput_ensure_entry_cxt(data, entry);
- entry->columns = pub_collist_to_bitmapset(entry->columns,
- cfdatum,
- entry->entry_cxt);
+ cols = pub_collist_to_bitmapset(cols, cfdatum,
+ entry->entry_cxt);
+
+ /*
+ * If column list includes all the columns of the table,
+ * set it to NULL.
+ */
+ if (bms_num_members(cols) == RelationGetNumberOfAttributes(relation))
+ {
+ bms_free(cols);
+ cols = NULL;
+ }
}
+
+ ReleaseSysCache(cftuple);
}
}
- /*
- * Found a publication with no column list, so we're done. But first
- * discard column list we might have from preceding publications.
- */
- if (pub_no_list)
+ if (first)
{
- if (cftuple)
- ReleaseSysCache(cftuple);
-
- bms_free(entry->columns);
- entry->columns = NULL;
-
- break;
+ entry->columns = cols;
+ first = false;
}
-
- ReleaseSysCache(cftuple);
+ else if (!bms_equal(entry->columns, cols))
+ ereport(ERROR,
+ errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot use different column lists for table \"%s.%s\" in different publications",
+ get_namespace_name(RelationGetNamespace(relation)),
+ RelationGetRelationName(relation)));
} /* loop all subscribed publications */
+
+ RelationClose(relation);
}
/*
diff --git a/src/test/subscription/t/031_column_list.pl b/src/test/subscription/t/031_column_list.pl
index 19812e1..22be6fd 100644
--- a/src/test/subscription/t/031_column_list.pl
+++ b/src/test/subscription/t/031_column_list.pl
@@ -361,13 +361,13 @@ is( $result, qq(1|abc
2|xyz), 'update on column tab2.c is not replicated');
-# TEST: add a table to two publications with different column lists, and
+# TEST: add a table to two publications with same column lists, and
# create a single subscription replicating both publications
$node_publisher->safe_psql(
'postgres', qq(
CREATE TABLE tab5 (a int PRIMARY KEY, b int, c int, d int);
CREATE PUBLICATION pub2 FOR TABLE tab5 (a, b);
- CREATE PUBLICATION pub3 FOR TABLE tab5 (a, d);
+ CREATE PUBLICATION pub3 FOR TABLE tab5 (a, b);
-- insert a couple initial records
INSERT INTO tab5 VALUES (1, 11, 111, 1111);
@@ -388,8 +388,7 @@ wait_for_subscription_sync($node_subscriber);
$node_publisher->wait_for_catchup('sub1');
-# insert data and make sure all the columns (union of the columns lists)
-# get fully replicated
+# insert data and make sure the columns in column list get fully replicated
$node_publisher->safe_psql(
'postgres', qq(
INSERT INTO tab5 VALUES (3, 33, 333, 3333);
@@ -399,42 +398,12 @@ $node_publisher->safe_psql(
$node_publisher->wait_for_catchup('sub1');
is( $node_subscriber->safe_psql('postgres', "SELECT * FROM tab5 ORDER BY a"),
- qq(1|11|1111
-2|22|2222
-3|33|3333
-4|44|4444),
+ qq(1|11|
+2|22|
+3|33|
+4|44|),
'overlapping publications with overlapping column lists');
-# and finally, remove the column list for one of the publications, which
-# means replicating all columns (removing the column list), but first add
-# the missing column to the table on subscriber
-$node_publisher->safe_psql(
- 'postgres', qq(
- ALTER PUBLICATION pub3 SET TABLE tab5;
-));
-
-$node_subscriber->safe_psql(
- 'postgres', qq(
- ALTER SUBSCRIPTION sub1 REFRESH PUBLICATION;
- ALTER TABLE tab5 ADD COLUMN c INT;
-));
-
-wait_for_subscription_sync($node_subscriber);
-
-$node_publisher->safe_psql(
- 'postgres', qq(
- INSERT INTO tab5 VALUES (5, 55, 555, 5555);
-));
-
-$node_publisher->wait_for_catchup('sub1');
-
-is( $node_subscriber->safe_psql('postgres', "SELECT * FROM tab5 ORDER BY a"),
- qq(1|11|1111|
-2|22|2222|
-3|33|3333|
-4|44|4444|
-5|55|5555|555),
- 'overlapping publications with overlapping column lists');
# TEST: create a table with a column list, then change the replica
# identity by replacing a primary key (but use a different column in
@@ -900,57 +869,21 @@ is( $node_subscriber->safe_psql(
3|),
'partitions with different replica identities not replicated correctly');
-# TEST: With a table included in multiple publications, we should use a
-# union of the column lists. So with column lists (a,b) and (a,c) we
-# should replicate (a,b,c).
-$node_publisher->safe_psql(
- 'postgres', qq(
- CREATE TABLE test_mix_1 (a int PRIMARY KEY, b int, c int);
- CREATE PUBLICATION pub_mix_1 FOR TABLE test_mix_1 (a, b);
- CREATE PUBLICATION pub_mix_2 FOR TABLE test_mix_1 (a, c);
-
- -- initial data
- INSERT INTO test_mix_1 VALUES (1, 2, 3);
-));
-
-$node_subscriber->safe_psql(
- 'postgres', qq(
- CREATE TABLE test_mix_1 (a int PRIMARY KEY, b int, c int);
- ALTER SUBSCRIPTION sub1 SET PUBLICATION pub_mix_1, pub_mix_2;
-));
-
-wait_for_subscription_sync($node_subscriber);
-
-$node_publisher->safe_psql(
- 'postgres', qq(
- INSERT INTO test_mix_1 VALUES (4, 5, 6);
-));
-
-$node_publisher->wait_for_catchup('sub1');
-
-is( $node_subscriber->safe_psql(
- 'postgres', "SELECT * FROM test_mix_1 ORDER BY a"),
- qq(1|2|3
-4|5|6),
- 'a mix of publications should use a union of column list');
-
-
-# TEST: With a table included in multiple publications, we should use a
-# union of the column lists. If any of the publications is FOR ALL
-# TABLES, we should replicate all columns.
+# TEST: With a table included in the publications which is FOR ALL TABLES, it
+# means replicate all columns.
# drop unnecessary tables, so as not to interfere with the FOR ALL TABLES
$node_publisher->safe_psql(
'postgres', qq(
- DROP TABLE tab1, tab2, tab3, tab4, tab5, tab6, tab7, test_mix_1,
+ DROP TABLE tab1, tab2, tab3, tab4, tab5, tab6, tab7,
test_part, test_part_a, test_part_b, test_part_c, test_part_d;
));
$node_publisher->safe_psql(
'postgres', qq(
CREATE TABLE test_mix_2 (a int PRIMARY KEY, b int, c int);
- CREATE PUBLICATION pub_mix_3 FOR TABLE test_mix_2 (a, b);
+ CREATE PUBLICATION pub_mix_3 FOR TABLE test_mix_2 (a, b, c);
CREATE PUBLICATION pub_mix_4 FOR ALL TABLES;
-- initial data
@@ -976,12 +909,11 @@ $node_publisher->wait_for_catchup('sub1');
is( $node_subscriber->safe_psql('postgres', "SELECT * FROM test_mix_2"),
qq(1|2|3
4|5|6),
- 'a mix of publications should use a union of column list');
+ 'all columns should be replicated');
-# TEST: With a table included in multiple publications, we should use a
-# union of the column lists. If any of the publications is FOR ALL
-# TABLES IN SCHEMA, we should replicate all columns.
+# TEST: With a table included in the publication which is FOR ALL TABLES
+# IN SCHEMA, it means replicate all columns.
$node_subscriber->safe_psql(
'postgres', qq(
@@ -993,7 +925,7 @@ $node_publisher->safe_psql(
'postgres', qq(
DROP TABLE test_mix_2;
CREATE TABLE test_mix_3 (a int PRIMARY KEY, b int, c int);
- CREATE PUBLICATION pub_mix_5 FOR TABLE test_mix_3 (a, b);
+ CREATE PUBLICATION pub_mix_5 FOR TABLE test_mix_3 (a, b, c);
CREATE PUBLICATION pub_mix_6 FOR ALL TABLES IN SCHEMA public;
-- initial data
@@ -1017,7 +949,7 @@ $node_publisher->wait_for_catchup('sub1');
is( $node_subscriber->safe_psql('postgres', "SELECT * FROM test_mix_3"),
qq(1|2|3
4|5|6),
- 'a mix of publications should use a union of column list');
+ 'all columns should be replicated');
# TEST: Check handling of publish_via_partition_root - if a partition is
@@ -1074,7 +1006,7 @@ is( $node_subscriber->safe_psql(
# TEST: Multiple publications which publish schema of parent table and
# partition. The partition is published through two publications, once
# through a schema (so no column list) containing the parent, and then
-# also directly (with a columns list). The expected outcome is there is
+# also directly (with all columns). The expected outcome is there is
# no column list.
$node_publisher->safe_psql(
@@ -1086,7 +1018,7 @@ $node_publisher->safe_psql(
CREATE TABLE t_1 PARTITION OF s1.t FOR VALUES FROM (1) TO (10);
CREATE PUBLICATION pub1 FOR ALL TABLES IN SCHEMA s1;
- CREATE PUBLICATION pub2 FOR TABLE t_1(b);
+ CREATE PUBLICATION pub2 FOR TABLE t_1(a, b, c);
-- initial data
INSERT INTO s1.t VALUES (1, 2, 3);
@@ -1232,7 +1164,6 @@ is( $node_subscriber->safe_psql(
4||),
'publication containing both parent and child relation');
-
$node_subscriber->stop('fast');
$node_publisher->stop('fast');
--
2.7.2.windows.1
[application/octet-stream] v2-0001-Extend-pg_publication_tables-to-display-column-list-.patch (14.8K, ../../OS0PR01MB5716D2CF3F2E39A6246A76CE94CF9@OS0PR01MB5716.jpnprd01.prod.outlook.com/3-v2-0001-Extend-pg_publication_tables-to-display-column-list-.patch)
download | inline diff:
From 11809a7d7052d0b894f55c22f0660a5fdc217082 Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Fri, 13 May 2022 11:35:14 +0800
Subject: [PATCH 1/2] Extend pg_publication_tables to display column list and
row filter.
Commit 923def9a53 and 52e4f0cd47 allowed to specify column lists and row
filters for publication tables. This commit extends the
pg_publication_tables view and pg_get_publication_tables function to
display that information.
This information will be useful to users and we also need this for the
later commit that prohibits combining multiple publications with different
column lists for the same table.
Author: Hou Zhijie
Reviewed By: Amit Kapila
Discussion: https://postgr.es/m/[email protected]
---
doc/src/sgml/catalogs.sgml | 27 +++++++++++++--
src/backend/catalog/pg_publication.c | 52 ++++++++++++++++++++++++++++-
src/backend/catalog/system_views.sql | 10 +++++-
src/backend/replication/logical/tablesync.c | 14 +++-----
src/include/catalog/pg_proc.dat | 8 ++---
src/test/regress/expected/publication.out | 42 +++++++++++------------
src/test/regress/expected/rules.out | 13 ++++++--
7 files changed, 124 insertions(+), 42 deletions(-)
diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index a533a21..008f81a 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -9691,7 +9691,7 @@ SCRAM-SHA-256$<replaceable><iteration count></replaceable>:<replaceable>&l
<row>
<entry><link linkend="view-pg-publication-tables"><structname>pg_publication_tables</structname></link></entry>
- <entry>publications and their associated tables</entry>
+ <entry>publications and information of their associated tables</entry>
</row>
<row>
@@ -11635,8 +11635,8 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
<para>
The view <structname>pg_publication_tables</structname> provides
- information about the mapping between publications and the tables they
- contain. Unlike the underlying catalog
+ information about the mapping between publications and information of
+ tables they contain. Unlike the underlying catalog
<link linkend="catalog-pg-publication-rel"><structname>pg_publication_rel</structname></link>,
this view expands publications defined as <literal>FOR ALL TABLES</literal>
and <literal>FOR ALL TABLES IN SCHEMA</literal>, so for such publications
@@ -11687,6 +11687,27 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
Name of table
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>attnames</structfield> <type>name[]</type>
+ (references <link linkend="catalog-pg-attribute"><structname>pg_attribute</structname></link>.<structfield>attname</structfield>)
+ </para>
+ <para>
+ Names of table columns included in the publication. This contains all
+ the columns of table when user didn't specify column list for the
+ table.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>rowfilter</structfield> <type>text</type>
+ </para>
+ <para>
+ Expression for the table's publication qualifying condition.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index e2c8bcb..3fe607e 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -1082,6 +1082,7 @@ get_publication_name(Oid pubid, bool missing_ok)
Datum
pg_get_publication_tables(PG_FUNCTION_ARGS)
{
+#define NUM_PUBLICATOIN_TABLES_ELEM 3
FuncCallContext *funcctx;
char *pubname = text_to_cstring(PG_GETARG_TEXT_PP(0));
Publication *publication;
@@ -1090,6 +1091,7 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
/* stuff done only on the first call of the function */
if (SRF_IS_FIRSTCALL())
{
+ TupleDesc tupdesc;
MemoryContext oldcontext;
/* create a function context for cross-call persistence */
@@ -1136,6 +1138,16 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
tables = filter_partitions(tables);
}
+ /* Construct a tuple descriptor for the result rows. */
+ tupdesc = CreateTemplateTupleDesc(NUM_PUBLICATOIN_TABLES_ELEM);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 1, "relid",
+ OIDOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 2, "attrs",
+ INT2VECTOROID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 3, "qual",
+ PG_NODE_TREEOID, -1, 0);
+
+ funcctx->tuple_desc = BlessTupleDesc(tupdesc);
funcctx->user_fctx = (void *) tables;
MemoryContextSwitchTo(oldcontext);
@@ -1147,9 +1159,47 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
if (funcctx->call_cntr < list_length(tables))
{
+ HeapTuple pubtuple = NULL;
+ HeapTuple rettuple;
Oid relid = list_nth_oid(tables, funcctx->call_cntr);
+ Datum values[NUM_PUBLICATOIN_TABLES_ELEM];
+ bool nulls[NUM_PUBLICATOIN_TABLES_ELEM];
+
+ /*
+ * Form tuple with appropriate data.
+ */
+ MemSet(nulls, 0, sizeof(nulls));
+ MemSet(values, 0, sizeof(values));
+
+ publication = GetPublicationByName(pubname, false);
+
+ values[0] = ObjectIdGetDatum(relid);
+
+ pubtuple = SearchSysCacheCopy2(PUBLICATIONRELMAP,
+ ObjectIdGetDatum(relid),
+ ObjectIdGetDatum(publication->oid));
+
+ if (HeapTupleIsValid(pubtuple))
+ {
+ /* Lookup the column list attribute. */
+ values[1] = SysCacheGetAttr(PUBLICATIONRELMAP, pubtuple,
+ Anum_pg_publication_rel_prattrs,
+ &(nulls[1]));
+
+ /* Null indicates no filter. */
+ values[2] = SysCacheGetAttr(PUBLICATIONRELMAP, pubtuple,
+ Anum_pg_publication_rel_prqual,
+ &(nulls[2]));
+ }
+ else
+ {
+ nulls[1] = true;
+ nulls[2] = true;
+ }
+
+ rettuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
- SRF_RETURN_NEXT(funcctx, ObjectIdGetDatum(relid));
+ SRF_RETURN_NEXT(funcctx, HeapTupleGetDatum(rettuple));
}
SRF_RETURN_DONE(funcctx);
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 0fc614e..fedaed5 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -368,7 +368,15 @@ CREATE VIEW pg_publication_tables AS
SELECT
P.pubname AS pubname,
N.nspname AS schemaname,
- C.relname AS tablename
+ C.relname AS tablename,
+ ( SELECT array_agg(a.attname ORDER BY a.attnum)
+ FROM unnest(CASE WHEN GPT.attrs IS NOT NULL THEN GPT.attrs
+ ELSE (SELECT array_agg(g) FROM generate_series(1, C.relnatts) g)
+ END) k
+ JOIN pg_attribute a
+ ON (a.attrelid = GPT.relid AND a.attnum = k)
+ ) AS attnames,
+ pg_get_expr(GPT.qual, GPT.relid) AS rowfilter
FROM pg_publication P,
LATERAL pg_get_publication_tables(P.pubname) GPT,
pg_class C JOIN pg_namespace N ON (N.oid = C.relnamespace)
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index b03e0f5..994c7a0 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -795,15 +795,12 @@ fetch_remote_table_info(char *nspname, char *relname,
resetStringInfo(&cmd);
appendStringInfo(&cmd,
"SELECT DISTINCT unnest"
- " FROM pg_publication p"
- " LEFT OUTER JOIN pg_publication_rel pr"
- " ON (p.oid = pr.prpubid AND pr.prrelid = %u)"
- " LEFT OUTER JOIN unnest(pr.prattrs) ON TRUE,"
+ " FROM pg_publication p,"
" LATERAL pg_get_publication_tables(p.pubname) gpt"
+ " LEFT OUTER JOIN unnest(gpt.attrs) ON TRUE"
" WHERE gpt.relid = %u"
" AND p.pubname IN ( %s )",
lrel->remoteid,
- lrel->remoteid,
pub_names.data);
pubres = walrcv_exec(LogRepWorkerWalRcvConn, cmd.data,
@@ -965,15 +962,12 @@ fetch_remote_table_info(char *nspname, char *relname,
/* Check for row filters. */
resetStringInfo(&cmd);
appendStringInfo(&cmd,
- "SELECT DISTINCT pg_get_expr(pr.prqual, pr.prrelid)"
- " FROM pg_publication p"
- " LEFT OUTER JOIN pg_publication_rel pr"
- " ON (p.oid = pr.prpubid AND pr.prrelid = %u),"
+ "SELECT DISTINCT pg_get_expr(gpt.qual, gpt.relid)"
+ " FROM pg_publication p,"
" LATERAL pg_get_publication_tables(p.pubname) gpt"
" WHERE gpt.relid = %u"
" AND p.pubname IN ( %s )",
lrel->remoteid,
- lrel->remoteid,
pub_names.data);
res = walrcv_exec(LogRepWorkerWalRcvConn, cmd.data, 1, qualRow);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index babe16f..87aa571 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -11673,11 +11673,11 @@
prosrc => 'pg_show_replication_origin_status' },
# publications
-{ oid => '6119', descr => 'get OIDs of tables in a publication',
+{ oid => '6119', descr => 'get information of tables in a publication',
proname => 'pg_get_publication_tables', prorows => '1000', proretset => 't',
- provolatile => 's', prorettype => 'oid', proargtypes => 'text',
- proallargtypes => '{text,oid}', proargmodes => '{i,o}',
- proargnames => '{pubname,relid}', prosrc => 'pg_get_publication_tables' },
+ provolatile => 's', prorettype => 'record', proargtypes => 'text',
+ proallargtypes => '{text,oid,int2vector,pg_node_tree}', proargmodes => '{i,o,o,o}',
+ proargnames => '{pubname,relid,attrs,qual}', prosrc => 'pg_get_publication_tables' },
{ oid => '6121',
descr => 'returns whether a relation can be part of a publication',
proname => 'pg_relation_is_publishable', provolatile => 's',
diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out
index 398c0f3..274b37d 100644
--- a/src/test/regress/expected/publication.out
+++ b/src/test/regress/expected/publication.out
@@ -1585,52 +1585,52 @@ CREATE TABLE sch2.tbl1_part1 PARTITION OF sch1.tbl1 FOR VALUES FROM (1) to (10);
-- Schema publication that does not include the schema that has the parent table
CREATE PUBLICATION pub FOR ALL TABLES IN SCHEMA sch2 WITH (PUBLISH_VIA_PARTITION_ROOT=1);
SELECT * FROM pg_publication_tables;
- pubname | schemaname | tablename
----------+------------+------------
- pub | sch2 | tbl1_part1
+ pubname | schemaname | tablename | attnames | rowfilter
+---------+------------+------------+----------+-----------
+ pub | sch2 | tbl1_part1 | {a} |
(1 row)
DROP PUBLICATION pub;
-- Table publication that does not include the parent table
CREATE PUBLICATION pub FOR TABLE sch2.tbl1_part1 WITH (PUBLISH_VIA_PARTITION_ROOT=1);
SELECT * FROM pg_publication_tables;
- pubname | schemaname | tablename
----------+------------+------------
- pub | sch2 | tbl1_part1
+ pubname | schemaname | tablename | attnames | rowfilter
+---------+------------+------------+----------+-----------
+ pub | sch2 | tbl1_part1 | {a} |
(1 row)
-- Table publication that includes both the parent table and the child table
ALTER PUBLICATION pub ADD TABLE sch1.tbl1;
SELECT * FROM pg_publication_tables;
- pubname | schemaname | tablename
----------+------------+-----------
- pub | sch1 | tbl1
+ pubname | schemaname | tablename | attnames | rowfilter
+---------+------------+-----------+----------+-----------
+ pub | sch1 | tbl1 | {a} |
(1 row)
DROP PUBLICATION pub;
-- Schema publication that does not include the schema that has the parent table
CREATE PUBLICATION pub FOR ALL TABLES IN SCHEMA sch2 WITH (PUBLISH_VIA_PARTITION_ROOT=0);
SELECT * FROM pg_publication_tables;
- pubname | schemaname | tablename
----------+------------+------------
- pub | sch2 | tbl1_part1
+ pubname | schemaname | tablename | attnames | rowfilter
+---------+------------+------------+----------+-----------
+ pub | sch2 | tbl1_part1 | {a} |
(1 row)
DROP PUBLICATION pub;
-- Table publication that does not include the parent table
CREATE PUBLICATION pub FOR TABLE sch2.tbl1_part1 WITH (PUBLISH_VIA_PARTITION_ROOT=0);
SELECT * FROM pg_publication_tables;
- pubname | schemaname | tablename
----------+------------+------------
- pub | sch2 | tbl1_part1
+ pubname | schemaname | tablename | attnames | rowfilter
+---------+------------+------------+----------+-----------
+ pub | sch2 | tbl1_part1 | {a} |
(1 row)
-- Table publication that includes both the parent table and the child table
ALTER PUBLICATION pub ADD TABLE sch1.tbl1;
SELECT * FROM pg_publication_tables;
- pubname | schemaname | tablename
----------+------------+------------
- pub | sch2 | tbl1_part1
+ pubname | schemaname | tablename | attnames | rowfilter
+---------+------------+------------+----------+-----------
+ pub | sch2 | tbl1_part1 | {a} |
(1 row)
DROP PUBLICATION pub;
@@ -1643,9 +1643,9 @@ CREATE TABLE sch1.tbl1_part3 (a int) PARTITION BY RANGE(a);
ALTER TABLE sch1.tbl1 ATTACH PARTITION sch1.tbl1_part3 FOR VALUES FROM (20) to (30);
CREATE PUBLICATION pub FOR ALL TABLES IN SCHEMA sch1 WITH (PUBLISH_VIA_PARTITION_ROOT=1);
SELECT * FROM pg_publication_tables;
- pubname | schemaname | tablename
----------+------------+-----------
- pub | sch1 | tbl1
+ pubname | schemaname | tablename | attnames | rowfilter
+---------+------------+-----------+----------+-----------
+ pub | sch1 | tbl1 | {a} |
(1 row)
RESET client_min_messages;
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 21effe8..fc3cde3 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1437,9 +1437,18 @@ pg_prepared_xacts| SELECT p.transaction,
LEFT JOIN pg_database d ON ((p.dbid = d.oid)));
pg_publication_tables| SELECT p.pubname,
n.nspname AS schemaname,
- c.relname AS tablename
+ c.relname AS tablename,
+ ( SELECT array_agg(a.attname ORDER BY a.attnum) AS array_agg
+ FROM (unnest(
+ CASE
+ WHEN (gpt.attrs IS NOT NULL) THEN (gpt.attrs)::integer[]
+ ELSE ( SELECT array_agg(g.g) AS array_agg
+ FROM generate_series(1, (c.relnatts)::integer) g(g))
+ END) k(k)
+ JOIN pg_attribute a ON (((a.attrelid = gpt.relid) AND (a.attnum = k.k))))) AS attnames,
+ pg_get_expr(gpt.qual, gpt.relid) AS rowfilter
FROM pg_publication p,
- LATERAL pg_get_publication_tables((p.pubname)::text) gpt(relid),
+ LATERAL pg_get_publication_tables((p.pubname)::text) gpt(relid, attrs, qual),
(pg_class c
JOIN pg_namespace n ON ((n.oid = c.relnamespace)))
WHERE (c.oid = gpt.relid);
--
2.7.2.windows.1
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-16 13:20 Alvaro Herrera <[email protected]>
parent: Amit Kapila <[email protected]>
1 sibling, 2 replies; 58+ messages in thread
From: Alvaro Herrera @ 2022-05-16 13:20 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; Tomas Vondra <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On 2022-May-16, Amit Kapila wrote:
> > Agreed. If we can get columnlist and rowfilter from pg_publication_tables, it
> > will be more convenient. And I think users that want to fetch the columnlist
> > and rowfilter of table can also benefit from it.
>
> After the change for this, we will give an error on combining
> publications where one of the publications specifies all columns in
> the table and the other doesn't provide any columns. We should not
> give an error as both mean all columns.
But don't we need to behave the same way for both column lists and row
filters? I understand that some cases with different row filters for
different publications have shown to have weird behavior, so I think
it'd make sense to restrict it in the same way. That would allow us to
extend the behavior in a sensible way when we develop that, instead of
setting in stone now behavior that we regret later.
> Few comments:
> =================
> 1.
> postgres=# select * from pg_publication_tables;
> pubname | schemaname | tablename | columnlist | rowfilter
> ---------+------------+-----------+------------+-----------
> pub1 | public | t1 | |
> pub2 | public | t1 | 1 2 | (c3 < 10)
> (2 rows)
>
> I think it is better to display column names for columnlist in the
> exposed view similar to attnames in the pg_stats_ext view. I think
> that will make it easier for users to understand this information.
+1
> I think we should change the "descr" to something like: 'get
> information of tables in a publication'
+1
> 3.
> +
> + /*
> + * We only throw a warning here so that the subcription can still be
> + * created and let user aware that something is going to fail later and
> + * they can fix the publications afterwards.
> + */
> + if (list_member(tablelist, rv))
> + ereport(WARNING,
> + errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
> + errmsg("cannot use different column lists for table \"%s.%s\" in
> different publications",
> + nspname, relname));
>
> Can we extend this comment to explain the case where after Alter
> Publication, if the user dumps and restores back the subscription,
> there is a possibility that "CREATE SUBSCRIPTION" won't work if we
> give ERROR here instead of WARNING?
Yeah, and not only the comment — I think we need to have more in the
warning message itself. How about:
ERROR: cannot use different column lists for table "..." in different publications
DETAIL: The subscription "..." cannot currently be used for replication.
I think this whole affair is a bit sad TBH and I'm sure it'll give us
some grief -- similar to replication slots becoming inactive and nobody
noticing. A user changing a publication in a way that prevents some
replica from working and the warnings are hidden, they could have
trouble noticing that the replica is stuck.
But I have no better ideas.
--
Álvaro Herrera 48°01'N 7°57'E — https://www.EnterpriseDB.com/
^ permalink raw reply [nested|flat] 58+ messages in thread
* RE: bogus: logical replication rows/cols combinations
@ 2022-05-17 03:25 [email protected] <[email protected]>
parent: [email protected] <[email protected]>
2 siblings, 0 replies; 58+ messages in thread
From: [email protected] @ 2022-05-17 03:25 UTC (permalink / raw)
To: [email protected] <[email protected]>; Amit Kapila <[email protected]>; +Cc: Tomas Vondra <[email protected]>; Alvaro Herrera <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On Mon, May 16, 2022 8:34 PM [email protected] <[email protected]> wrote:
>
> Attach the new version patch.
>
Thanks for your patch. Here are some comments:
1. (0001 patch)
/*
* Returns Oids of tables in a publication.
*/
Datum
pg_get_publication_tables(PG_FUNCTION_ARGS)
Should we modify the comment of pg_get_publication_tables() to "Returns
information of tables in a publication"?
2. (0002 patch)
+ * Note that we don't support the case where column list is different for
+ * the same table when combining publications. But we still need to check
+ * all the given publication-table mappings and report an error if any
+ * publications have different column list.
*
* Multiple publications might have multiple column lists for this
* relation.
I think it would be better if we swap the order of these two paragraphs.
Regards,
Shi yu
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-17 03:26 Amit Kapila <[email protected]>
parent: Alvaro Herrera <[email protected]>
1 sibling, 0 replies; 58+ messages in thread
From: Amit Kapila @ 2022-05-17 03:26 UTC (permalink / raw)
To: Alvaro Herrera <[email protected]>; +Cc: [email protected] <[email protected]>; Tomas Vondra <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On Mon, May 16, 2022 at 6:50 PM Alvaro Herrera <[email protected]> wrote:
>
> On 2022-May-16, Amit Kapila wrote:
>
> > > Agreed. If we can get columnlist and rowfilter from pg_publication_tables, it
> > > will be more convenient. And I think users that want to fetch the columnlist
> > > and rowfilter of table can also benefit from it.
> >
> > After the change for this, we will give an error on combining
> > publications where one of the publications specifies all columns in
> > the table and the other doesn't provide any columns. We should not
> > give an error as both mean all columns.
>
> But don't we need to behave the same way for both column lists and row
> filters? I understand that some cases with different row filters for
> different publications have shown to have weird behavior, so I think
> it'd make sense to restrict it in the same way.
>
I think the case where we are worried about row filter behavior is for
initial table sync where we ignore publication actions and that is
true with and without row filters. See email [1]. We are planning to
document that behavior as a separate patch. The idea we have used for
row filters is similar to what IBM DB2 [2] and Oracle [3] uses where
they allow combining filters with pub-action (operation (insert,
update, delete) in their case).
I think both column lists and row filters have a different purpose and
we shouldn't try to make them behave in the same way. The main purpose
of introducing a column list is to have statically different shapes on
publisher and subscriber or hide sensitive column data. In both cases,
it doesn't seem to make sense to combine column lists and we didn't
find any other database doing so. OTOH, for row filters, it makes
sense to combine filters for each pub-action as both IBM DB2 and
Oracle seems to be doing.
>
> > 3.
> > +
> > + /*
> > + * We only throw a warning here so that the subcription can still be
> > + * created and let user aware that something is going to fail later and
> > + * they can fix the publications afterwards.
> > + */
> > + if (list_member(tablelist, rv))
> > + ereport(WARNING,
> > + errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
> > + errmsg("cannot use different column lists for table \"%s.%s\" in
> > different publications",
> > + nspname, relname));
> >
> > Can we extend this comment to explain the case where after Alter
> > Publication, if the user dumps and restores back the subscription,
> > there is a possibility that "CREATE SUBSCRIPTION" won't work if we
> > give ERROR here instead of WARNING?
>
> Yeah, and not only the comment — I think we need to have more in the
> warning message itself.
>
But as mentioned by Hou-San in his last email (pg_dump of subscription
always set (connect = false) which means it won't try to fetch column
list), I think we don't need to give a WARNING here, instead, we can
use ERROR. So, do we need the extra DETAIL (The subscription "..."
cannot currently be used for replication.) as that is implicit for the
ERROR case?
>
> I think this whole affair is a bit sad TBH and I'm sure it'll give us
> some grief -- similar to replication slots becoming inactive and nobody
> noticing. A user changing a publication in a way that prevents some
> replica from working and the warnings are hidden, they could have
> trouble noticing that the replica is stuck.
>
I agree and it seems this is the best we can do for now.
[1] - https://www.postgresql.org/message-id/CAA4eK1L_98LF7Db4yFY1PhKKRzoT83xtN41jTS5X%2B8OeGrAkLw%40mail.g...
[2] - https://www.ibm.com/docs/en/idr/11.4.0?topic=rows-log-record-variables
[3] - https://docs.oracle.com/en/cloud/paas/goldengate-cloud/gwuad/selecting-and-filtering-rows.html#GUID-...
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 58+ messages in thread
* RE: bogus: logical replication rows/cols combinations
@ 2022-05-17 06:49 [email protected] <[email protected]>
parent: [email protected] <[email protected]>
2 siblings, 0 replies; 58+ messages in thread
From: [email protected] @ 2022-05-17 06:49 UTC (permalink / raw)
To: [email protected] <[email protected]>; Amit Kapila <[email protected]>; +Cc: Tomas Vondra <[email protected]>; Alvaro Herrera <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On Monday, May 16, 2022 9:34 PM [email protected] <[email protected]> wrote:
> Attach the new version patch.
Hi,
I have few minor comments.
For v2-0001.
(1) Unnecessary period at the end of column explanation
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>rowfilter</structfield> <type>text</type>
+ </para>
+ <para>
+ Expression for the table's publication qualifying condition.
+ </para></entry>
+ </row>
It seems when we write a simple noun to explain a column,
we don't need to put a period at the end of the explanation.
Kindly change
FROM:
"Expression for the table's publication qualifying condition."
TO:
"Expression for the table's publication qualifying condition"
For v2-0002.
(a) typo in the commit message
Kindly change
FROM:
"In both cases, it doesn't seems to make sense to combine column lists."
TO:
"In both cases, it doesn't seem to make sense to combine column lists."
or "In both cases, it doesn't make sense to combine column lists."
(b) fetch_table_list
+ if (list_member(tablelist, rv))
+ ereport(ERROR,
+ errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot use different column lists for table \"%s.%s\" in different publications",
+ nspname, relname));
Kindly add tests for new error paths, when we add them.
Best Regards,
Takamichi Osumi
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-17 06:52 Amit Kapila <[email protected]>
parent: [email protected] <[email protected]>
2 siblings, 1 reply; 58+ messages in thread
From: Amit Kapila @ 2022-05-17 06:52 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Tomas Vondra <[email protected]>; Alvaro Herrera <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On Mon, May 16, 2022 at 6:04 PM [email protected]
<[email protected]> wrote:
>
> Attach the new version patch.
>
Few minor comments:
==================
1.
+ <para>
+ Names of table columns included in the publication. This contains all
+ the columns of table when user didn't specify column list for the
+ table.
+ </para></entry>
Can we slightly change it to: "Names of table columns included in the
publication. This contains all the columns of the table when the user
didn't specify the column list for the table."
2. Below comments needs to be removed from tablesync.c as we don't
combine column lists after this patch.
* For initial synchronization, column lists can be ignored in following
* cases:
*
* 1) one of the subscribed publications for the table hasn't specified
* any column list
*
* 2) one of the subscribed publications has puballtables set to true
*
* 3) one of the subscribed publications is declared as ALL TABLES IN
* SCHEMA that includes this relation
3.
Note that we don't support the case where column list is different for
+ * the same table when combining publications. But we still need to check
+ * all the given publication-table mappings and report an error if any
+ * publications have different column list.
Can we change this comment to: "Note that we don't support the case
where the column list is different for the same table when combining
publications. But one can later change the publication so we still
need to check all the given publication-table mappings and report an
error if any publications have a different column list."?
4. Can we add a test for different column lists if it is not already there?
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 58+ messages in thread
* RE: bogus: logical replication rows/cols combinations
@ 2022-05-17 09:10 [email protected] <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 1 reply; 58+ messages in thread
From: [email protected] @ 2022-05-17 09:10 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; +Cc: Tomas Vondra <[email protected]>; Alvaro Herrera <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On Tuesday, May 17, 2022 2:53 PM Amit Kapila <[email protected]> wrote:
>
> Few minor comments:
> ==================
> 1.
> + <para>
> + Names of table columns included in the publication. This contains all
> + the columns of table when user didn't specify column list for the
> + table.
> + </para></entry>
>
> Can we slightly change it to: "Names of table columns included in the
> publication. This contains all the columns of the table when the user
> didn't specify the column list for the table."
>
> 2. Below comments needs to be removed from tablesync.c as we don't
> combine column lists after this patch.
>
> * For initial synchronization, column lists can be ignored in following
> * cases:
> *
> * 1) one of the subscribed publications for the table hasn't specified
> * any column list
> *
> * 2) one of the subscribed publications has puballtables set to true
> *
> * 3) one of the subscribed publications is declared as ALL TABLES IN
> * SCHEMA that includes this relation
>
> 3.
> Note that we don't support the case where column list is different for
> + * the same table when combining publications. But we still need to check
> + * all the given publication-table mappings and report an error if any
> + * publications have different column list.
>
> Can we change this comment to: "Note that we don't support the case
> where the column list is different for the same table when combining
> publications. But one can later change the publication so we still
> need to check all the given publication-table mappings and report an
> error if any publications have a different column list."?
>
> 4. Can we add a test for different column lists if it is not already there?
Thanks for the comments.
Attach the new version patch which addressed all the above comments and
comments from Shi yu[1] and Osumi-san[2].
[1] https://www.postgresql.org/message-id/OSZPR01MB6310F32344884F9C12F45071FDCE9%40OSZPR01MB6310.jpnprd0...
[2] https://www.postgresql.org/message-id/TYCPR01MB83736AEC2493FCBB75CC7556EDCE9%40TYCPR01MB8373.jpnprd0...
Best regards,
Hou zj
Attachments:
[application/octet-stream] v3-0001-Extend-pg_publication_tables-to-display-column-list-.patch (15.0K, ../../OS0PR01MB571647F2739F710E3D1B045D94CE9@OS0PR01MB5716.jpnprd01.prod.outlook.com/2-v3-0001-Extend-pg_publication_tables-to-display-column-list-.patch)
download | inline diff:
From 87104fadcbdba46874a52e72e844ed4c3a73de1d Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Fri, 13 May 2022 11:35:14 +0800
Subject: [PATCH] Extend pg_publication_tables to display column list and row
filter.
Commit 923def9a53 and 52e4f0cd47 allowed to specify column lists and row
filters for publication tables. This commit extends the
pg_publication_tables view and pg_get_publication_tables function to
display that information.
This information will be useful to users and we also need this for the
later commit that prohibits combining multiple publications with different
column lists for the same table.
Author: Hou Zhijie
Reviewed By: Amit Kapila, Alvaro Herrera, Shi Yu, Takamichi Osumi
Discussion: https://postgr.es/m/[email protected]
---
doc/src/sgml/catalogs.sgml | 27 +++++++++++++--
src/backend/catalog/pg_publication.c | 54 +++++++++++++++++++++++++++--
src/backend/catalog/system_views.sql | 10 +++++-
src/backend/replication/logical/tablesync.c | 14 +++-----
src/include/catalog/pg_proc.dat | 8 ++---
src/test/regress/expected/publication.out | 42 +++++++++++-----------
src/test/regress/expected/rules.out | 13 +++++--
7 files changed, 125 insertions(+), 43 deletions(-)
diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index a533a21..d96c72e 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -9691,7 +9691,7 @@ SCRAM-SHA-256$<replaceable><iteration count></replaceable>:<replaceable>&l
<row>
<entry><link linkend="view-pg-publication-tables"><structname>pg_publication_tables</structname></link></entry>
- <entry>publications and their associated tables</entry>
+ <entry>publications and information of their associated tables</entry>
</row>
<row>
@@ -11635,8 +11635,8 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
<para>
The view <structname>pg_publication_tables</structname> provides
- information about the mapping between publications and the tables they
- contain. Unlike the underlying catalog
+ information about the mapping between publications and information of
+ tables they contain. Unlike the underlying catalog
<link linkend="catalog-pg-publication-rel"><structname>pg_publication_rel</structname></link>,
this view expands publications defined as <literal>FOR ALL TABLES</literal>
and <literal>FOR ALL TABLES IN SCHEMA</literal>, so for such publications
@@ -11687,6 +11687,27 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
Name of table
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>attnames</structfield> <type>name[]</type>
+ (references <link linkend="catalog-pg-attribute"><structname>pg_attribute</structname></link>.<structfield>attname</structfield>)
+ </para>
+ <para>
+ Names of table columns included in the publication. This contains all
+ the columns of the table when the user didn't specify the column list
+ for the table.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>rowfilter</structfield> <type>text</type>
+ </para>
+ <para>
+ Expression for the table's publication qualifying condition
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index e2c8bcb..152b4ba 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -1077,11 +1077,12 @@ get_publication_name(Oid pubid, bool missing_ok)
}
/*
- * Returns Oids of tables in a publication.
+ * Returns information of tables in a publication.
*/
Datum
pg_get_publication_tables(PG_FUNCTION_ARGS)
{
+#define NUM_PUBLICATOIN_TABLES_ELEM 3
FuncCallContext *funcctx;
char *pubname = text_to_cstring(PG_GETARG_TEXT_PP(0));
Publication *publication;
@@ -1090,6 +1091,7 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
/* stuff done only on the first call of the function */
if (SRF_IS_FIRSTCALL())
{
+ TupleDesc tupdesc;
MemoryContext oldcontext;
/* create a function context for cross-call persistence */
@@ -1136,6 +1138,16 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
tables = filter_partitions(tables);
}
+ /* Construct a tuple descriptor for the result rows. */
+ tupdesc = CreateTemplateTupleDesc(NUM_PUBLICATOIN_TABLES_ELEM);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 1, "relid",
+ OIDOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 2, "attrs",
+ INT2VECTOROID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 3, "qual",
+ PG_NODE_TREEOID, -1, 0);
+
+ funcctx->tuple_desc = BlessTupleDesc(tupdesc);
funcctx->user_fctx = (void *) tables;
MemoryContextSwitchTo(oldcontext);
@@ -1147,9 +1159,47 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
if (funcctx->call_cntr < list_length(tables))
{
+ HeapTuple pubtuple = NULL;
+ HeapTuple rettuple;
Oid relid = list_nth_oid(tables, funcctx->call_cntr);
+ Datum values[NUM_PUBLICATOIN_TABLES_ELEM];
+ bool nulls[NUM_PUBLICATOIN_TABLES_ELEM];
+
+ /*
+ * Form tuple with appropriate data.
+ */
+ MemSet(nulls, 0, sizeof(nulls));
+ MemSet(values, 0, sizeof(values));
+
+ publication = GetPublicationByName(pubname, false);
+
+ values[0] = ObjectIdGetDatum(relid);
+
+ pubtuple = SearchSysCacheCopy2(PUBLICATIONRELMAP,
+ ObjectIdGetDatum(relid),
+ ObjectIdGetDatum(publication->oid));
+
+ if (HeapTupleIsValid(pubtuple))
+ {
+ /* Lookup the column list attribute. */
+ values[1] = SysCacheGetAttr(PUBLICATIONRELMAP, pubtuple,
+ Anum_pg_publication_rel_prattrs,
+ &(nulls[1]));
+
+ /* Null indicates no filter. */
+ values[2] = SysCacheGetAttr(PUBLICATIONRELMAP, pubtuple,
+ Anum_pg_publication_rel_prqual,
+ &(nulls[2]));
+ }
+ else
+ {
+ nulls[1] = true;
+ nulls[2] = true;
+ }
+
+ rettuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
- SRF_RETURN_NEXT(funcctx, ObjectIdGetDatum(relid));
+ SRF_RETURN_NEXT(funcctx, HeapTupleGetDatum(rettuple));
}
SRF_RETURN_DONE(funcctx);
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 0fc614e..fedaed5 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -368,7 +368,15 @@ CREATE VIEW pg_publication_tables AS
SELECT
P.pubname AS pubname,
N.nspname AS schemaname,
- C.relname AS tablename
+ C.relname AS tablename,
+ ( SELECT array_agg(a.attname ORDER BY a.attnum)
+ FROM unnest(CASE WHEN GPT.attrs IS NOT NULL THEN GPT.attrs
+ ELSE (SELECT array_agg(g) FROM generate_series(1, C.relnatts) g)
+ END) k
+ JOIN pg_attribute a
+ ON (a.attrelid = GPT.relid AND a.attnum = k)
+ ) AS attnames,
+ pg_get_expr(GPT.qual, GPT.relid) AS rowfilter
FROM pg_publication P,
LATERAL pg_get_publication_tables(P.pubname) GPT,
pg_class C JOIN pg_namespace N ON (N.oid = C.relnamespace)
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index b03e0f5..994c7a0 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -795,15 +795,12 @@ fetch_remote_table_info(char *nspname, char *relname,
resetStringInfo(&cmd);
appendStringInfo(&cmd,
"SELECT DISTINCT unnest"
- " FROM pg_publication p"
- " LEFT OUTER JOIN pg_publication_rel pr"
- " ON (p.oid = pr.prpubid AND pr.prrelid = %u)"
- " LEFT OUTER JOIN unnest(pr.prattrs) ON TRUE,"
+ " FROM pg_publication p,"
" LATERAL pg_get_publication_tables(p.pubname) gpt"
+ " LEFT OUTER JOIN unnest(gpt.attrs) ON TRUE"
" WHERE gpt.relid = %u"
" AND p.pubname IN ( %s )",
lrel->remoteid,
- lrel->remoteid,
pub_names.data);
pubres = walrcv_exec(LogRepWorkerWalRcvConn, cmd.data,
@@ -965,15 +962,12 @@ fetch_remote_table_info(char *nspname, char *relname,
/* Check for row filters. */
resetStringInfo(&cmd);
appendStringInfo(&cmd,
- "SELECT DISTINCT pg_get_expr(pr.prqual, pr.prrelid)"
- " FROM pg_publication p"
- " LEFT OUTER JOIN pg_publication_rel pr"
- " ON (p.oid = pr.prpubid AND pr.prrelid = %u),"
+ "SELECT DISTINCT pg_get_expr(gpt.qual, gpt.relid)"
+ " FROM pg_publication p,"
" LATERAL pg_get_publication_tables(p.pubname) gpt"
" WHERE gpt.relid = %u"
" AND p.pubname IN ( %s )",
lrel->remoteid,
- lrel->remoteid,
pub_names.data);
res = walrcv_exec(LogRepWorkerWalRcvConn, cmd.data, 1, qualRow);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index babe16f..87aa571 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -11673,11 +11673,11 @@
prosrc => 'pg_show_replication_origin_status' },
# publications
-{ oid => '6119', descr => 'get OIDs of tables in a publication',
+{ oid => '6119', descr => 'get information of tables in a publication',
proname => 'pg_get_publication_tables', prorows => '1000', proretset => 't',
- provolatile => 's', prorettype => 'oid', proargtypes => 'text',
- proallargtypes => '{text,oid}', proargmodes => '{i,o}',
- proargnames => '{pubname,relid}', prosrc => 'pg_get_publication_tables' },
+ provolatile => 's', prorettype => 'record', proargtypes => 'text',
+ proallargtypes => '{text,oid,int2vector,pg_node_tree}', proargmodes => '{i,o,o,o}',
+ proargnames => '{pubname,relid,attrs,qual}', prosrc => 'pg_get_publication_tables' },
{ oid => '6121',
descr => 'returns whether a relation can be part of a publication',
proname => 'pg_relation_is_publishable', provolatile => 's',
diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out
index 398c0f3..274b37d 100644
--- a/src/test/regress/expected/publication.out
+++ b/src/test/regress/expected/publication.out
@@ -1585,52 +1585,52 @@ CREATE TABLE sch2.tbl1_part1 PARTITION OF sch1.tbl1 FOR VALUES FROM (1) to (10);
-- Schema publication that does not include the schema that has the parent table
CREATE PUBLICATION pub FOR ALL TABLES IN SCHEMA sch2 WITH (PUBLISH_VIA_PARTITION_ROOT=1);
SELECT * FROM pg_publication_tables;
- pubname | schemaname | tablename
----------+------------+------------
- pub | sch2 | tbl1_part1
+ pubname | schemaname | tablename | attnames | rowfilter
+---------+------------+------------+----------+-----------
+ pub | sch2 | tbl1_part1 | {a} |
(1 row)
DROP PUBLICATION pub;
-- Table publication that does not include the parent table
CREATE PUBLICATION pub FOR TABLE sch2.tbl1_part1 WITH (PUBLISH_VIA_PARTITION_ROOT=1);
SELECT * FROM pg_publication_tables;
- pubname | schemaname | tablename
----------+------------+------------
- pub | sch2 | tbl1_part1
+ pubname | schemaname | tablename | attnames | rowfilter
+---------+------------+------------+----------+-----------
+ pub | sch2 | tbl1_part1 | {a} |
(1 row)
-- Table publication that includes both the parent table and the child table
ALTER PUBLICATION pub ADD TABLE sch1.tbl1;
SELECT * FROM pg_publication_tables;
- pubname | schemaname | tablename
----------+------------+-----------
- pub | sch1 | tbl1
+ pubname | schemaname | tablename | attnames | rowfilter
+---------+------------+-----------+----------+-----------
+ pub | sch1 | tbl1 | {a} |
(1 row)
DROP PUBLICATION pub;
-- Schema publication that does not include the schema that has the parent table
CREATE PUBLICATION pub FOR ALL TABLES IN SCHEMA sch2 WITH (PUBLISH_VIA_PARTITION_ROOT=0);
SELECT * FROM pg_publication_tables;
- pubname | schemaname | tablename
----------+------------+------------
- pub | sch2 | tbl1_part1
+ pubname | schemaname | tablename | attnames | rowfilter
+---------+------------+------------+----------+-----------
+ pub | sch2 | tbl1_part1 | {a} |
(1 row)
DROP PUBLICATION pub;
-- Table publication that does not include the parent table
CREATE PUBLICATION pub FOR TABLE sch2.tbl1_part1 WITH (PUBLISH_VIA_PARTITION_ROOT=0);
SELECT * FROM pg_publication_tables;
- pubname | schemaname | tablename
----------+------------+------------
- pub | sch2 | tbl1_part1
+ pubname | schemaname | tablename | attnames | rowfilter
+---------+------------+------------+----------+-----------
+ pub | sch2 | tbl1_part1 | {a} |
(1 row)
-- Table publication that includes both the parent table and the child table
ALTER PUBLICATION pub ADD TABLE sch1.tbl1;
SELECT * FROM pg_publication_tables;
- pubname | schemaname | tablename
----------+------------+------------
- pub | sch2 | tbl1_part1
+ pubname | schemaname | tablename | attnames | rowfilter
+---------+------------+------------+----------+-----------
+ pub | sch2 | tbl1_part1 | {a} |
(1 row)
DROP PUBLICATION pub;
@@ -1643,9 +1643,9 @@ CREATE TABLE sch1.tbl1_part3 (a int) PARTITION BY RANGE(a);
ALTER TABLE sch1.tbl1 ATTACH PARTITION sch1.tbl1_part3 FOR VALUES FROM (20) to (30);
CREATE PUBLICATION pub FOR ALL TABLES IN SCHEMA sch1 WITH (PUBLISH_VIA_PARTITION_ROOT=1);
SELECT * FROM pg_publication_tables;
- pubname | schemaname | tablename
----------+------------+-----------
- pub | sch1 | tbl1
+ pubname | schemaname | tablename | attnames | rowfilter
+---------+------------+-----------+----------+-----------
+ pub | sch1 | tbl1 | {a} |
(1 row)
RESET client_min_messages;
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 21effe8..fc3cde3 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1437,9 +1437,18 @@ pg_prepared_xacts| SELECT p.transaction,
LEFT JOIN pg_database d ON ((p.dbid = d.oid)));
pg_publication_tables| SELECT p.pubname,
n.nspname AS schemaname,
- c.relname AS tablename
+ c.relname AS tablename,
+ ( SELECT array_agg(a.attname ORDER BY a.attnum) AS array_agg
+ FROM (unnest(
+ CASE
+ WHEN (gpt.attrs IS NOT NULL) THEN (gpt.attrs)::integer[]
+ ELSE ( SELECT array_agg(g.g) AS array_agg
+ FROM generate_series(1, (c.relnatts)::integer) g(g))
+ END) k(k)
+ JOIN pg_attribute a ON (((a.attrelid = gpt.relid) AND (a.attnum = k.k))))) AS attnames,
+ pg_get_expr(gpt.qual, gpt.relid) AS rowfilter
FROM pg_publication p,
- LATERAL pg_get_publication_tables((p.pubname)::text) gpt(relid),
+ LATERAL pg_get_publication_tables((p.pubname)::text) gpt(relid, attrs, qual),
(pg_class c
JOIN pg_namespace n ON ((n.oid = c.relnamespace)))
WHERE (c.oid = gpt.relid);
--
2.7.2.windows.1
[application/octet-stream] v3-0002-Prohibit-combining-publications-with-different-colum.patch (22.0K, ../../OS0PR01MB571647F2739F710E3D1B045D94CE9@OS0PR01MB5716.jpnprd01.prod.outlook.com/3-v3-0002-Prohibit-combining-publications-with-different-colum.patch)
download | inline diff:
From 91a4774b9f351cb09f461ee920be9031cc4e073e Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Fri, 13 May 2022 13:19:49 +0800
Subject: [PATCH] Prohibit combining publications with different column lists.
The main purpose of introducing a column list is to have statically
different shapes on publisher and subscriber or hide sensitive column
data. In both cases, it doesn't seem to make sense to combine column
lists.
So, we disallow the cases where the column list is different for the same
table when combining publications. It can be later extended to combine the
column lists for selective cases where required.
Reported-by: Alvaro Herrera
Author: Hou Zhijie
Reviewed-by: Amit Kapila, Alvaro Herrera, Shi Yu, Takamichi Osumi
Discussion: https://postgr.es/m/[email protected]
---
doc/src/sgml/ref/alter_publication.sgml | 12 ++-
doc/src/sgml/ref/create_subscription.sgml | 5 +
src/backend/commands/subscriptioncmds.c | 23 ++++-
src/backend/replication/logical/tablesync.c | 72 ++++++-------
src/backend/replication/pgoutput/pgoutput.c | 80 +++++++--------
src/test/subscription/t/031_column_list.pl | 150 ++++++++++++----------------
6 files changed, 176 insertions(+), 166 deletions(-)
diff --git a/doc/src/sgml/ref/alter_publication.sgml b/doc/src/sgml/ref/alter_publication.sgml
index e2cce49..f03933a 100644
--- a/doc/src/sgml/ref/alter_publication.sgml
+++ b/doc/src/sgml/ref/alter_publication.sgml
@@ -116,7 +116,17 @@ ALTER PUBLICATION <replaceable class="parameter">name</replaceable> RENAME TO <r
<para>
Optionally, a column list can be specified. See <xref
- linkend="sql-createpublication"/> for details.
+ linkend="sql-createpublication"/> for details. Note that a subscription
+ having several publications in which the same table has been published
+ with different column lists is not supported. So, changing the column
+ lists of the tables being subscribed could cause inconsistency of column
+ lists among publications in which case <command>ALTER PUBLICATION</command>
+ command will be successful but later the WalSender in publisher or the
+ subscriber may throw an error. In this scenario, the user needs to
+ recreate the subscription after adjusting the column list or drop the
+ problematic publication using
+ <literal>ALTER SUBSCRIPTION ... DROP PUBLICATION</literal> and then add
+ it back after adjusting the column list.
</para>
<para>
diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml
index 203bb41..f6f82a0 100644
--- a/doc/src/sgml/ref/create_subscription.sgml
+++ b/doc/src/sgml/ref/create_subscription.sgml
@@ -356,6 +356,11 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl
</para>
<para>
+ Subscription having several publications in which the same table has been
+ published with different column lists is not supported.
+ </para>
+
+ <para>
We allow non-existent publications to be specified so that users can add
those later. This means
<link linkend="catalog-pg-subscription"><structname>pg_subscription</structname></link>
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 690cdaa..991b2c1 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1753,7 +1753,8 @@ AlterSubscriptionOwner_oid(Oid subid, Oid newOwnerId)
/*
* Get the list of tables which belong to specified publications on the
- * publisher connection.
+ * publisher connection. Also get the column list for each table and check if
+ * column lists are the same in different publications.
*/
static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
@@ -1761,17 +1762,18 @@ fetch_table_list(WalReceiverConn *wrconn, List *publications)
WalRcvExecResult *res;
StringInfoData cmd;
TupleTableSlot *slot;
- Oid tableRow[2] = {TEXTOID, TEXTOID};
+ Oid tableRow[3] = {TEXTOID, TEXTOID, NAMEARRAYOID};
List *tablelist = NIL;
initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename\n"
+ appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename, \n"
+ " t.attnames\n"
" FROM pg_catalog.pg_publication_tables t\n"
" WHERE t.pubname IN (");
get_publications_str(publications, &cmd, true);
appendStringInfoChar(&cmd, ')');
- res = walrcv_exec(wrconn, cmd.data, 2, tableRow);
+ res = walrcv_exec(wrconn, cmd.data, 3, tableRow);
pfree(cmd.data);
if (res->status != WALRCV_OK_TUPLES)
@@ -1795,7 +1797,18 @@ fetch_table_list(WalReceiverConn *wrconn, List *publications)
Assert(!isnull);
rv = makeRangeVar(nspname, relname, -1);
- tablelist = lappend(tablelist, rv);
+
+ /*
+ * We don't support the case where column list is different for the
+ * same table in different publications.
+ */
+ if (list_member(tablelist, rv))
+ ereport(ERROR,
+ errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot use different column lists for table \"%s.%s\" in different publications",
+ nspname, relname));
+ else
+ tablelist = lappend(tablelist, rv);
ExecClearTuple(slot);
}
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 994c7a0..afc5bcd 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -753,17 +753,6 @@ fetch_remote_table_info(char *nspname, char *relname,
/*
* Get column lists for each relation.
*
- * For initial synchronization, column lists can be ignored in following
- * cases:
- *
- * 1) one of the subscribed publications for the table hasn't specified
- * any column list
- *
- * 2) one of the subscribed publications has puballtables set to true
- *
- * 3) one of the subscribed publications is declared as ALL TABLES IN
- * SCHEMA that includes this relation
- *
* We need to do this before fetching info about column names and types,
* so that we can skip columns that should not be replicated.
*/
@@ -771,7 +760,7 @@ fetch_remote_table_info(char *nspname, char *relname,
{
WalRcvExecResult *pubres;
TupleTableSlot *slot;
- Oid attrsRow[] = {INT2OID};
+ Oid attrsRow[] = {INT2VECTOROID};
StringInfoData pub_names;
bool first = true;
@@ -786,19 +775,17 @@ fetch_remote_table_info(char *nspname, char *relname,
/*
* Fetch info about column lists for the relation (from all the
- * publications). We unnest the int2vector values, because that makes
- * it easier to combine lists by simply adding the attnums to a new
- * bitmap (without having to parse the int2vector data). This
- * preserves NULL values, so that if one of the publications has no
- * column list, we'll know that.
+ * publications).
*/
resetStringInfo(&cmd);
appendStringInfo(&cmd,
- "SELECT DISTINCT unnest"
+ "SELECT DISTINCT"
+ " (CASE WHEN (array_length(gpt.attrs, 1) = c.relnatts)"
+ " THEN NULL ELSE gpt.attrs END)"
" FROM pg_publication p,"
- " LATERAL pg_get_publication_tables(p.pubname) gpt"
- " LEFT OUTER JOIN unnest(gpt.attrs) ON TRUE"
- " WHERE gpt.relid = %u"
+ " LATERAL pg_get_publication_tables(p.pubname) gpt,"
+ " pg_class c"
+ " WHERE gpt.relid = %u AND c.oid = gpt.relid"
" AND p.pubname IN ( %s )",
lrel->remoteid,
pub_names.data);
@@ -813,26 +800,43 @@ fetch_remote_table_info(char *nspname, char *relname,
nspname, relname, pubres->err)));
/*
- * Merge the column lists (from different publications) by creating a
- * single bitmap with all the attnums. If we find a NULL value, that
- * means one of the publications has no column list for the table
- * we're syncing.
+ * We don't support the case where the column list is different for the
+ * same table when combining publications. So there should be only one
+ * row returned. Although we already checked this when creating
+ * subscription, we still need to check here in case the column list
+ * was changed after creating the subscription and before the sync
+ * worker is started.
+ */
+ if (tuplestore_tuple_count(pubres->tuplestore) > 1)
+ ereport(ERROR,
+ errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot use different column lists for table \"%s.%s\" in different publications",
+ nspname, relname));
+
+ /*
+ * Get the column list and build a single bitmap with the attnums.
+ *
+ * If we find a NULL value, it means all the columns should be
+ * replicated.
*/
slot = MakeSingleTupleTableSlot(pubres->tupledesc, &TTSOpsMinimalTuple);
- while (tuplestore_gettupleslot(pubres->tuplestore, true, false, slot))
+ if (tuplestore_gettupleslot(pubres->tuplestore, true, false, slot))
{
Datum cfval = slot_getattr(slot, 1, &isnull);
- /* NULL means empty column list, so we're done. */
- if (isnull)
+ if (!isnull)
{
- bms_free(included_cols);
- included_cols = NULL;
- break;
- }
+ ArrayType *arr;
+ int nelems;
+ int16 *elems;
- included_cols = bms_add_member(included_cols,
- DatumGetInt16(cfval));
+ arr = DatumGetArrayTypeP(cfval);
+ nelems = ARR_DIMS(arr)[0];
+ elems = (int16 *) ARR_DATA_PTR(arr);
+
+ for (natt = 0; natt < nelems; natt++)
+ included_cols = bms_add_member(included_cols, elems[natt]);
+ }
ExecClearTuple(slot);
}
diff --git a/src/backend/replication/pgoutput/pgoutput.c b/src/backend/replication/pgoutput/pgoutput.c
index 42c06af..e9079bd 100644
--- a/src/backend/replication/pgoutput/pgoutput.c
+++ b/src/backend/replication/pgoutput/pgoutput.c
@@ -979,30 +979,31 @@ pgoutput_column_list_init(PGOutputData *data, List *publications,
RelationSyncEntry *entry)
{
ListCell *lc;
+ bool first = true;
+ Relation relation = RelationIdGetRelation(entry->publish_as_relid);
/*
* Find if there are any column lists for this relation. If there are,
- * build a bitmap merging all the column lists.
- *
- * All the given publication-table mappings must be checked.
+ * build a bitmap using the column lists.
*
* Multiple publications might have multiple column lists for this
* relation.
*
+ * Note that we don't support the case where the column list is different
+ * for the same table when combining publications. But one can later change
+ * the publication so we still need to check all the given
+ * publication-table mappings and report an error if any publications have
+ * a different column list.
+ *
* FOR ALL TABLES and FOR ALL TABLES IN SCHEMA implies "don't use column
- * list" so it takes precedence.
+ * list".
*/
foreach(lc, publications)
{
Publication *pub = lfirst(lc);
HeapTuple cftuple = NULL;
Datum cfdatum = 0;
-
- /*
- * Assume there's no column list. Only if we find pg_publication_rel
- * entry with a column list we'll switch it to false.
- */
- bool pub_no_list = true;
+ Bitmapset *cols = NULL;
/*
* If the publication is FOR ALL TABLES then it is treated the same as
@@ -1011,6 +1012,8 @@ pgoutput_column_list_init(PGOutputData *data, List *publications,
*/
if (!pub->alltables)
{
+ bool pub_no_list = true;
+
/*
* Check for the presence of a column list in this publication.
*
@@ -1024,51 +1027,48 @@ pgoutput_column_list_init(PGOutputData *data, List *publications,
if (HeapTupleIsValid(cftuple))
{
- /*
- * Lookup the column list attribute.
- *
- * Note: We update the pub_no_list value directly, because if
- * the value is NULL, we have no list (and vice versa).
- */
+ /* Lookup the column list attribute. */
cfdatum = SysCacheGetAttr(PUBLICATIONRELMAP, cftuple,
Anum_pg_publication_rel_prattrs,
&pub_no_list);
- /*
- * Build the column list bitmap in the per-entry context.
- *
- * We need to merge column lists from all publications, so we
- * update the same bitmapset. If the column list is null, we
- * interpret it as replicating all columns.
- */
+ /* Build the column list bitmap in the per-entry context. */
if (!pub_no_list) /* when not null */
{
pgoutput_ensure_entry_cxt(data, entry);
- entry->columns = pub_collist_to_bitmapset(entry->columns,
- cfdatum,
- entry->entry_cxt);
+ cols = pub_collist_to_bitmapset(cols, cfdatum,
+ entry->entry_cxt);
+
+ /*
+ * If column list includes all the columns of the table,
+ * set it to NULL.
+ */
+ if (bms_num_members(cols) == RelationGetNumberOfAttributes(relation))
+ {
+ bms_free(cols);
+ cols = NULL;
+ }
}
+
+ ReleaseSysCache(cftuple);
}
}
- /*
- * Found a publication with no column list, so we're done. But first
- * discard column list we might have from preceding publications.
- */
- if (pub_no_list)
+ if (first)
{
- if (cftuple)
- ReleaseSysCache(cftuple);
-
- bms_free(entry->columns);
- entry->columns = NULL;
-
- break;
+ entry->columns = cols;
+ first = false;
}
-
- ReleaseSysCache(cftuple);
+ else if (!bms_equal(entry->columns, cols))
+ ereport(ERROR,
+ errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot use different column lists for table \"%s.%s\" in different publications",
+ get_namespace_name(RelationGetNamespace(relation)),
+ RelationGetRelationName(relation)));
} /* loop all subscribed publications */
+
+ RelationClose(relation);
}
/*
diff --git a/src/test/subscription/t/031_column_list.pl b/src/test/subscription/t/031_column_list.pl
index 19812e1..7f031bc 100644
--- a/src/test/subscription/t/031_column_list.pl
+++ b/src/test/subscription/t/031_column_list.pl
@@ -20,6 +20,7 @@ $node_subscriber->append_conf('postgresql.conf',
$node_subscriber->start;
my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres';
+my $offset = 0;
sub wait_for_subscription_sync
{
@@ -361,13 +362,13 @@ is( $result, qq(1|abc
2|xyz), 'update on column tab2.c is not replicated');
-# TEST: add a table to two publications with different column lists, and
+# TEST: add a table to two publications with same column lists, and
# create a single subscription replicating both publications
$node_publisher->safe_psql(
'postgres', qq(
CREATE TABLE tab5 (a int PRIMARY KEY, b int, c int, d int);
CREATE PUBLICATION pub2 FOR TABLE tab5 (a, b);
- CREATE PUBLICATION pub3 FOR TABLE tab5 (a, d);
+ CREATE PUBLICATION pub3 FOR TABLE tab5 (a, b);
-- insert a couple initial records
INSERT INTO tab5 VALUES (1, 11, 111, 1111);
@@ -388,8 +389,7 @@ wait_for_subscription_sync($node_subscriber);
$node_publisher->wait_for_catchup('sub1');
-# insert data and make sure all the columns (union of the columns lists)
-# get fully replicated
+# insert data and make sure the columns in column list get fully replicated
$node_publisher->safe_psql(
'postgres', qq(
INSERT INTO tab5 VALUES (3, 33, 333, 3333);
@@ -399,42 +399,12 @@ $node_publisher->safe_psql(
$node_publisher->wait_for_catchup('sub1');
is( $node_subscriber->safe_psql('postgres', "SELECT * FROM tab5 ORDER BY a"),
- qq(1|11|1111
-2|22|2222
-3|33|3333
-4|44|4444),
+ qq(1|11|
+2|22|
+3|33|
+4|44|),
'overlapping publications with overlapping column lists');
-# and finally, remove the column list for one of the publications, which
-# means replicating all columns (removing the column list), but first add
-# the missing column to the table on subscriber
-$node_publisher->safe_psql(
- 'postgres', qq(
- ALTER PUBLICATION pub3 SET TABLE tab5;
-));
-
-$node_subscriber->safe_psql(
- 'postgres', qq(
- ALTER SUBSCRIPTION sub1 REFRESH PUBLICATION;
- ALTER TABLE tab5 ADD COLUMN c INT;
-));
-
-wait_for_subscription_sync($node_subscriber);
-
-$node_publisher->safe_psql(
- 'postgres', qq(
- INSERT INTO tab5 VALUES (5, 55, 555, 5555);
-));
-
-$node_publisher->wait_for_catchup('sub1');
-
-is( $node_subscriber->safe_psql('postgres', "SELECT * FROM tab5 ORDER BY a"),
- qq(1|11|1111|
-2|22|2222|
-3|33|3333|
-4|44|4444|
-5|55|5555|555),
- 'overlapping publications with overlapping column lists');
# TEST: create a table with a column list, then change the replica
# identity by replacing a primary key (but use a different column in
@@ -900,57 +870,21 @@ is( $node_subscriber->safe_psql(
3|),
'partitions with different replica identities not replicated correctly');
-# TEST: With a table included in multiple publications, we should use a
-# union of the column lists. So with column lists (a,b) and (a,c) we
-# should replicate (a,b,c).
-
-$node_publisher->safe_psql(
- 'postgres', qq(
- CREATE TABLE test_mix_1 (a int PRIMARY KEY, b int, c int);
- CREATE PUBLICATION pub_mix_1 FOR TABLE test_mix_1 (a, b);
- CREATE PUBLICATION pub_mix_2 FOR TABLE test_mix_1 (a, c);
-
- -- initial data
- INSERT INTO test_mix_1 VALUES (1, 2, 3);
-));
-
-$node_subscriber->safe_psql(
- 'postgres', qq(
- CREATE TABLE test_mix_1 (a int PRIMARY KEY, b int, c int);
- ALTER SUBSCRIPTION sub1 SET PUBLICATION pub_mix_1, pub_mix_2;
-));
-
-wait_for_subscription_sync($node_subscriber);
-
-$node_publisher->safe_psql(
- 'postgres', qq(
- INSERT INTO test_mix_1 VALUES (4, 5, 6);
-));
-
-$node_publisher->wait_for_catchup('sub1');
-
-is( $node_subscriber->safe_psql(
- 'postgres', "SELECT * FROM test_mix_1 ORDER BY a"),
- qq(1|2|3
-4|5|6),
- 'a mix of publications should use a union of column list');
-
-# TEST: With a table included in multiple publications, we should use a
-# union of the column lists. If any of the publications is FOR ALL
-# TABLES, we should replicate all columns.
+# TEST: With a table included in the publications which is FOR ALL TABLES, it
+# means replicate all columns.
# drop unnecessary tables, so as not to interfere with the FOR ALL TABLES
$node_publisher->safe_psql(
'postgres', qq(
- DROP TABLE tab1, tab2, tab3, tab4, tab5, tab6, tab7, test_mix_1,
+ DROP TABLE tab1, tab2, tab3, tab4, tab5, tab6, tab7,
test_part, test_part_a, test_part_b, test_part_c, test_part_d;
));
$node_publisher->safe_psql(
'postgres', qq(
CREATE TABLE test_mix_2 (a int PRIMARY KEY, b int, c int);
- CREATE PUBLICATION pub_mix_3 FOR TABLE test_mix_2 (a, b);
+ CREATE PUBLICATION pub_mix_3 FOR TABLE test_mix_2 (a, b, c);
CREATE PUBLICATION pub_mix_4 FOR ALL TABLES;
-- initial data
@@ -976,12 +910,11 @@ $node_publisher->wait_for_catchup('sub1');
is( $node_subscriber->safe_psql('postgres', "SELECT * FROM test_mix_2"),
qq(1|2|3
4|5|6),
- 'a mix of publications should use a union of column list');
+ 'all columns should be replicated');
-# TEST: With a table included in multiple publications, we should use a
-# union of the column lists. If any of the publications is FOR ALL
-# TABLES IN SCHEMA, we should replicate all columns.
+# TEST: With a table included in the publication which is FOR ALL TABLES
+# IN SCHEMA, it means replicate all columns.
$node_subscriber->safe_psql(
'postgres', qq(
@@ -993,7 +926,7 @@ $node_publisher->safe_psql(
'postgres', qq(
DROP TABLE test_mix_2;
CREATE TABLE test_mix_3 (a int PRIMARY KEY, b int, c int);
- CREATE PUBLICATION pub_mix_5 FOR TABLE test_mix_3 (a, b);
+ CREATE PUBLICATION pub_mix_5 FOR TABLE test_mix_3 (a, b, c);
CREATE PUBLICATION pub_mix_6 FOR ALL TABLES IN SCHEMA public;
-- initial data
@@ -1017,7 +950,7 @@ $node_publisher->wait_for_catchup('sub1');
is( $node_subscriber->safe_psql('postgres', "SELECT * FROM test_mix_3"),
qq(1|2|3
4|5|6),
- 'a mix of publications should use a union of column list');
+ 'all columns should be replicated');
# TEST: Check handling of publish_via_partition_root - if a partition is
@@ -1074,7 +1007,7 @@ is( $node_subscriber->safe_psql(
# TEST: Multiple publications which publish schema of parent table and
# partition. The partition is published through two publications, once
# through a schema (so no column list) containing the parent, and then
-# also directly (with a columns list). The expected outcome is there is
+# also directly (with all columns). The expected outcome is there is
# no column list.
$node_publisher->safe_psql(
@@ -1086,7 +1019,7 @@ $node_publisher->safe_psql(
CREATE TABLE t_1 PARTITION OF s1.t FOR VALUES FROM (1) TO (10);
CREATE PUBLICATION pub1 FOR ALL TABLES IN SCHEMA s1;
- CREATE PUBLICATION pub2 FOR TABLE t_1(b);
+ CREATE PUBLICATION pub2 FOR TABLE t_1(a, b, c);
-- initial data
INSERT INTO s1.t VALUES (1, 2, 3);
@@ -1233,6 +1166,51 @@ is( $node_subscriber->safe_psql(
'publication containing both parent and child relation');
+# TEST: With a table included in multiple publications with different column
+# lists, we should catch the error when creating the subscription.
+
+$node_publisher->safe_psql('postgres', qq(
+ CREATE TABLE test_mix_1 (a int PRIMARY KEY, b int, c int);
+ CREATE PUBLICATION pub_mix_1 FOR TABLE test_mix_1 (a, b);
+ CREATE PUBLICATION pub_mix_2 FOR TABLE test_mix_1 (a, c);
+));
+
+$node_subscriber->safe_psql('postgres', qq(
+ DROP SUBSCRIPTION sub1;
+ CREATE TABLE test_mix_1 (a int PRIMARY KEY, b int, c int);
+));
+
+my ($cmdret, $stdout, $stderr) = $node_subscriber->psql(
+ 'postgres', qq(
+ CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_mix_1, pub_mix_2;
+));
+
+ok( $stderr =~
+ qr/cannot use different column lists for table "public.test_mix_1" in different publications/,
+ 'different column lists detected');
+
+# TEST: If the column list is changed after creating the subscription, we
+# should catch the error reported by walsender.
+
+$node_publisher->safe_psql('postgres', qq(
+ ALTER PUBLICATION pub_mix_1 SET TABLE test_mix_1 (a, c);
+));
+
+$node_subscriber->safe_psql('postgres', qq(
+ CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_mix_1, pub_mix_2;
+));
+
+$node_publisher->wait_for_catchup('sub1');
+
+$node_publisher->safe_psql('postgres', qq(
+ ALTER PUBLICATION pub_mix_1 SET TABLE test_mix_1 (a, b);
+ INSERT INTO test_mix_1 VALUES(1, 1, 1);
+));
+
+$offset = $node_publisher->wait_for_log(
+ qr/cannot use different column lists for table "public.test_mix_1" in different publications/,
+ $offset);
+
$node_subscriber->stop('fast');
$node_publisher->stop('fast');
--
2.7.2.windows.1
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-18 02:28 Amit Kapila <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 0 replies; 58+ messages in thread
From: Amit Kapila @ 2022-05-18 02:28 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; Tomas Vondra <[email protected]>; Alvaro Herrera <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On Tue, May 17, 2022 at 2:40 PM [email protected]
<[email protected]> wrote:
>
> Attach the new version patch which addressed all the above comments and
> comments from Shi yu[1] and Osumi-san[2].
>
Thanks, your first patch looks good to me. I'll commit that tomorrow
unless there are more comments on the same. The second one is also in
good shape but I would like to test it a bit more and also see if
others have any suggestions/objections on the same.
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-19 05:03 Amit Kapila <[email protected]>
parent: Alvaro Herrera <[email protected]>
1 sibling, 1 reply; 58+ messages in thread
From: Amit Kapila @ 2022-05-19 05:03 UTC (permalink / raw)
To: Alvaro Herrera <[email protected]>; +Cc: [email protected] <[email protected]>; Tomas Vondra <[email protected]>; Peter Eisentraut <[email protected]>; Pg Hackers <[email protected]>
On Mon, May 16, 2022 at 6:50 PM Alvaro Herrera <[email protected]> wrote:
>
> On 2022-May-16, Amit Kapila wrote:
>
> > Few comments:
> > =================
> > 1.
> > postgres=# select * from pg_publication_tables;
> > pubname | schemaname | tablename | columnlist | rowfilter
> > ---------+------------+-----------+------------+-----------
> > pub1 | public | t1 | |
> > pub2 | public | t1 | 1 2 | (c3 < 10)
> > (2 rows)
> >
> > I think it is better to display column names for columnlist in the
> > exposed view similar to attnames in the pg_stats_ext view. I think
> > that will make it easier for users to understand this information.
>
> +1
>
I have committed the first patch after fixing this part. It seems Tom
is not very happy doing this after beta-1 [1]. The reason we get this
information via this view (and underlying function) is that it
simplifies the queries on the subscriber-side as you can see in the
second patch. The query change is as below:
@@ -1761,17 +1762,18 @@ fetch_table_list(WalReceiverConn *wrconn, List
*publications)
WalRcvExecResult *res;
StringInfoData cmd;
TupleTableSlot *slot;
- Oid tableRow[2] = {TEXTOID, TEXTOID};
+ Oid tableRow[3] = {TEXTOID, TEXTOID, NAMEARRAYOID};
List *tablelist = NIL;
initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename\n"
+ appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename, \n"
+ " t.attnames\n"
" FROM pg_catalog.pg_publication_tables t\n"
" WHERE t.pubname IN (");
Now, there is another way to change this query as well as done by
Hou-San in his first version [2] of the patch. The changed query with
that approach will be something like:
@@ -1761,17 +1762,34 @@ fetch_table_list(WalReceiverConn *wrconn, List
*publications)
WalRcvExecResult *res;
StringInfoData cmd;
TupleTableSlot *slot;
- Oid tableRow[2] = {TEXTOID, TEXTOID};
+ Oid tableRow[3] = {TEXTOID, TEXTOID, INT2VECTOROID};
List *tablelist = NIL;
initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename\n"
- " FROM pg_catalog.pg_publication_tables t\n"
+ appendStringInfoString(&cmd,
+ "SELECT DISTINCT t.schemaname,\n"
+ " t.tablename,\n"
+ " (CASE WHEN (array_length(pr.prattrs, 1) = t.relnatts)\n"
+ " THEN NULL ELSE pr.prattrs END)\n"
+ " FROM (SELECT P.pubname AS pubname,\n"
+ " N.nspname AS schemaname,\n"
+ " C.relname AS tablename,\n"
+ " P.oid AS pubid,\n"
+ " C.oid AS reloid,\n"
+ " C.relnatts\n"
+ " FROM pg_publication P,\n"
+ " LATERAL pg_get_publication_tables(P.pubname) GPT,\n"
+ " pg_class C JOIN pg_namespace N\n"
+ " ON (N.oid = C.relnamespace)\n"
+ " WHERE C.oid = GPT.relid) t\n"
+ " LEFT OUTER JOIN pg_publication_rel pr\n"
+ " ON (t.pubid = pr.prpubid AND\n"
+ " pr.prrelid = reloid)\n"
It appeared slightly complex and costly to me, so I have given the
suggestion to change it as we have now in the second patch as shown
above. Now, I can think of below ways to proceed here:
a. Revert the change in view (and underlying function) as done in
commit 0ff20288e1 and consider the alternate way (using a slightly
complex query) to fix. Then maybe for PG-16, we can simplify it by
changing the underlying function and view.
b. Proceed with the current approach of using a simplified query.
What do you think?
[1] - https://www.postgresql.org/message-id/91075.1652929852%40sss.pgh.pa.us
[2] - https://www.postgresql.org/message-id/OS0PR01MB5716A594C58DE4FFD1F8100B94C89%40OS0PR01MB5716.jpnprd0...
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-19 12:07 Justin Pryzby <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 1 reply; 58+ messages in thread
From: Justin Pryzby @ 2022-05-19 12:07 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; [email protected]; Tomas Vondra <[email protected]>; Peter Eisentraut <[email protected]>; [email protected]
On Thu, May 19, 2022 at 10:33:13AM +0530, Amit Kapila wrote:
> I have committed the first patch after fixing this part. It seems Tom
> is not very happy doing this after beta-1 [1]. The reason we get this
> information via this view (and underlying function) is that it
> simplifies the queries on the subscriber-side as you can see in the
> second patch. The query change is as below:
> [1] - https://www.postgresql.org/message-id/91075.1652929852%40sss.pgh.pa.us
I think Tom's concern is that adding information to a view seems like adding a
feature that hadn't previously been contemplated.
(Catalog changes themselves are not prohibited during the beta period).
> a. Revert the change in view (and underlying function) as done in
> commit 0ff20288e1 and consider the alternate way (using a slightly
> complex query) to fix. Then maybe for PG-16, we can simplify it by
> changing the underlying function and view.
But, ISTM that it makes no sense to do it differently for v15 just to avoid the
appearance of adding a new feature, only to re-do it in 2 weeks for v16...
So (from a passive observer) +0.1 to keep the current patch.
I have some minor language fixes to that patch.
diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index d96c72e5310..82aa84e96e1 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -9691,7 +9691,7 @@ SCRAM-SHA-256$<replaceable><iteration count></replaceable>:<replaceable>&l
<row>
<entry><link linkend="view-pg-publication-tables"><structname>pg_publication_tables</structname></link></entry>
- <entry>publications and information of their associated tables</entry>
+ <entry>publications and information about their associated tables</entry>
</row>
<row>
@@ -11635,7 +11635,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
<para>
The view <structname>pg_publication_tables</structname> provides
- information about the mapping between publications and information of
+ information about the mapping between publications and information about
tables they contain. Unlike the underlying catalog
<link linkend="catalog-pg-publication-rel"><structname>pg_publication_rel</structname></link>,
this view expands publications defined as <literal>FOR ALL TABLES</literal>
@@ -11695,7 +11695,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
</para>
<para>
Names of table columns included in the publication. This contains all
- the columns of the table when the user didn't specify the column list
+ the columns of the table when the user didn't specify a column list
for the table.
</para></entry>
</row>
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index 8c7fca62de3..2f706f638ce 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -1077,7 +1077,7 @@ get_publication_name(Oid pubid, bool missing_ok)
}
/*
- * Returns information of tables in a publication.
+ * Returns information about tables in a publication.
*/
Datum
pg_get_publication_tables(PG_FUNCTION_ARGS)
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 87aa571a331..86f13293090 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -11673,7 +11673,7 @@
prosrc => 'pg_show_replication_origin_status' },
# publications
-{ oid => '6119', descr => 'get information of tables in a publication',
+{ oid => '6119', descr => 'get information about tables in a publication',
proname => 'pg_get_publication_tables', prorows => '1000', proretset => 't',
provolatile => 's', prorettype => 'record', proargtypes => 'text',
proallargtypes => '{text,oid,int2vector,pg_node_tree}', proargmodes => '{i,o,o,o}',
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-19 14:24 Tom Lane <[email protected]>
parent: Justin Pryzby <[email protected]>
0 siblings, 1 reply; 58+ messages in thread
From: Tom Lane @ 2022-05-19 14:24 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: Amit Kapila <[email protected]>; Alvaro Herrera <[email protected]>; [email protected]; Tomas Vondra <[email protected]>; Peter Eisentraut <[email protected]>; [email protected]
Justin Pryzby <[email protected]> writes:
> On Thu, May 19, 2022 at 10:33:13AM +0530, Amit Kapila wrote:
>> I have committed the first patch after fixing this part. It seems Tom
>> is not very happy doing this after beta-1 [1]. The reason we get this
>> information via this view (and underlying function) is that it
>> simplifies the queries on the subscriber-side as you can see in the
>> second patch. The query change is as below:
>> [1] - https://www.postgresql.org/message-id/91075.1652929852%40sss.pgh.pa.us
> I think Tom's concern is that adding information to a view seems like adding a
> feature that hadn't previously been contemplated.
> (Catalog changes themselves are not prohibited during the beta period).
It certainly smells like a new feature, but my concern was more around the
post-beta catalog change. We do those only if really forced to, and the
explanation in the commit message didn't satisfy me as to why it was
necessary. This explanation isn't much better --- if we're trying to
prohibit a certain class of publication definitions, what good does it do
to check that on the subscriber side? Even more to the point, how can we
have a subscriber do that by relying on view columns that don't exist in
older versions? I'm also quite concerned about anything that involves
subscribers examining row filter conditions; that sounds like a pretty
direct route to bugs involving unsolvability and the halting problem.
(But I've not read very much of this thread ... been a bit under the
weather the last couple weeks. Maybe this actually is a sane solution.
It just doesn't sound like one at this level of detail.)
regards, tom lane
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-20 03:06 Amit Kapila <[email protected]>
parent: Tom Lane <[email protected]>
0 siblings, 2 replies; 58+ messages in thread
From: Amit Kapila @ 2022-05-20 03:06 UTC (permalink / raw)
To: Tom Lane <[email protected]>; +Cc: Justin Pryzby <[email protected]>; Alvaro Herrera <[email protected]>; Hou, Zhijie/侯 志杰 <[email protected]>; Tomas Vondra <[email protected]>; Peter Eisentraut <[email protected]>; PostgreSQL Hackers <[email protected]>
On Thu, May 19, 2022 at 7:54 PM Tom Lane <[email protected]> wrote:
>
> Justin Pryzby <[email protected]> writes:
> > On Thu, May 19, 2022 at 10:33:13AM +0530, Amit Kapila wrote:
> >> I have committed the first patch after fixing this part. It seems Tom
> >> is not very happy doing this after beta-1 [1]. The reason we get this
> >> information via this view (and underlying function) is that it
> >> simplifies the queries on the subscriber-side as you can see in the
> >> second patch. The query change is as below:
> >> [1] - https://www.postgresql.org/message-id/91075.1652929852%40sss.pgh.pa.us
>
> > I think Tom's concern is that adding information to a view seems like adding a
> > feature that hadn't previously been contemplated.
> > (Catalog changes themselves are not prohibited during the beta period).
>
> It certainly smells like a new feature, but my concern was more around the
> post-beta catalog change. We do those only if really forced to, and the
> explanation in the commit message didn't satisfy me as to why it was
> necessary. This explanation isn't much better --- if we're trying to
> prohibit a certain class of publication definitions, what good does it do
> to check that on the subscriber side?
>
It is required on the subscriber side because prohibition is only for
the cases where multiple publications are combined. We disallow the
cases where the column list is different for the same table when
combining publications. For example:
Publisher-side:
Create table tab(c1 int, c2 int);
Create Publication pub1 for table tab(c1);
Create Publication pub1 for table tab(c2);
Subscriber-side:
Create Subscription sub1 Connection 'dbname=postgres' Publication pub1, pub2;
We want to prohibit such cases. So, it would be better to check at the
time of 'Create Subscription' to validate such combinations and
prohibit them. To achieve that we extended the existing function
pg_get_publication_tables() and view pg_publication_tables to expose
the column list and verify such a combination. We primarily need
column list information for this prohibition but it appeared natural
to expose the row filter.
As mentioned in my previous email, we can fetch the required
information directly from system table pg_publication_rel and extend
the query in fetch_table_list to achieve the desired purpose but
extending the existing function/view for this appears to be a simpler
way.
> Even more to the point, how can we
> have a subscriber do that by relying on view columns that don't exist in
> older versions?
>
We need a version check like (if
(walrcv_server_version(LogRepWorkerWalRcvConn) >= 150000)) for that.
> I'm also quite concerned about anything that involves
> subscribers examining row filter conditions; that sounds like a pretty
> direct route to bugs involving unsolvability and the halting problem.
>
We examine only the column list for the purpose of this prohibition.
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 58+ messages in thread
* RE: bogus: logical replication rows/cols combinations
@ 2022-05-24 05:33 [email protected] <[email protected]>
parent: Amit Kapila <[email protected]>
1 sibling, 1 reply; 58+ messages in thread
From: [email protected] @ 2022-05-24 05:33 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: Justin Pryzby <[email protected]>; Alvaro Herrera <[email protected]>; Tomas Vondra <[email protected]>; Peter Eisentraut <[email protected]>; PostgreSQL Hackers <[email protected]>; Tom Lane <[email protected]>
On Friday, May 20, 2022 11:06 AM Amit Kapila <[email protected]> wrote:
>
> On Thu, May 19, 2022 at 7:54 PM Tom Lane <[email protected]> wrote:
>
> > Even more to the point, how can we
> > have a subscriber do that by relying on view columns that don't exist
> > in older versions?
> >
>
> We need a version check like (if
> (walrcv_server_version(LogRepWorkerWalRcvConn) >= 150000)) for that.
Thanks for pointing it out. Here is the new version patch which add this version check.
Best regards,
Hou zj
Attachments:
[application/octet-stream] v4-0002-Prohibit-combining-publications-with-different-colum.patch (22.3K, ../../OS0PR01MB5716AD7C0FE7386630BDBAAB94D79@OS0PR01MB5716.jpnprd01.prod.outlook.com/2-v4-0002-Prohibit-combining-publications-with-different-colum.patch)
download | inline diff:
From 080d3e8ac5d9ec44b52f0a03e6d204d7f451a779 Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Fri, 13 May 2022 13:19:49 +0800
Subject: [PATCH] Prohibit combining publications with different column lists.
The main purpose of introducing a column list is to have statically
different shapes on publisher and subscriber or hide sensitive column
data. In both cases, it doesn't seem to make sense to combine column
lists.
So, we disallow the cases where the column list is different for the same
table when combining publications. It can be later extended to combine the
column lists for selective cases where required.
Reported-by: Alvaro Herrera
Author: Hou Zhijie
Reviewed-by: Amit Kapila, Alvaro Herrera, Shi Yu, Takamichi Osumi
Discussion: https://postgr.es/m/[email protected]
---
doc/src/sgml/ref/alter_publication.sgml | 12 ++-
doc/src/sgml/ref/create_subscription.sgml | 5 +
src/backend/commands/subscriptioncmds.c | 30 ++++--
src/backend/replication/logical/tablesync.c | 72 ++++++-------
src/backend/replication/pgoutput/pgoutput.c | 80 +++++++--------
src/test/subscription/t/031_column_list.pl | 150 ++++++++++++----------------
6 files changed, 182 insertions(+), 167 deletions(-)
diff --git a/doc/src/sgml/ref/alter_publication.sgml b/doc/src/sgml/ref/alter_publication.sgml
index e2cce49..f03933a 100644
--- a/doc/src/sgml/ref/alter_publication.sgml
+++ b/doc/src/sgml/ref/alter_publication.sgml
@@ -116,7 +116,17 @@ ALTER PUBLICATION <replaceable class="parameter">name</replaceable> RENAME TO <r
<para>
Optionally, a column list can be specified. See <xref
- linkend="sql-createpublication"/> for details.
+ linkend="sql-createpublication"/> for details. Note that a subscription
+ having several publications in which the same table has been published
+ with different column lists is not supported. So, changing the column
+ lists of the tables being subscribed could cause inconsistency of column
+ lists among publications in which case <command>ALTER PUBLICATION</command>
+ command will be successful but later the WalSender in publisher or the
+ subscriber may throw an error. In this scenario, the user needs to
+ recreate the subscription after adjusting the column list or drop the
+ problematic publication using
+ <literal>ALTER SUBSCRIPTION ... DROP PUBLICATION</literal> and then add
+ it back after adjusting the column list.
</para>
<para>
diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml
index 203bb41..f6f82a0 100644
--- a/doc/src/sgml/ref/create_subscription.sgml
+++ b/doc/src/sgml/ref/create_subscription.sgml
@@ -356,6 +356,11 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl
</para>
<para>
+ Subscription having several publications in which the same table has been
+ published with different column lists is not supported.
+ </para>
+
+ <para>
We allow non-existent publications to be specified so that users can add
those later. This means
<link linkend="catalog-pg-subscription"><structname>pg_subscription</structname></link>
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 690cdaa..641caf7 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1753,7 +1753,8 @@ AlterSubscriptionOwner_oid(Oid subid, Oid newOwnerId)
/*
* Get the list of tables which belong to specified publications on the
- * publisher connection.
+ * publisher connection. Also get the column list for each table and check if
+ * column lists are the same in different publications.
*/
static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
@@ -1761,17 +1762,23 @@ fetch_table_list(WalReceiverConn *wrconn, List *publications)
WalRcvExecResult *res;
StringInfoData cmd;
TupleTableSlot *slot;
- Oid tableRow[2] = {TEXTOID, TEXTOID};
+ Oid tableRow[3] = {TEXTOID, TEXTOID, NAMEARRAYOID};
List *tablelist = NIL;
+ bool check_columnlist = (walrcv_server_version(wrconn) >= 150000);
initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename\n"
- " FROM pg_catalog.pg_publication_tables t\n"
+ appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename \n");
+
+ /* Get column lists for each relation if the publisher supports it */
+ if (check_columnlist)
+ appendStringInfoString(&cmd, ", t.attnames\n");
+
+ appendStringInfoString(&cmd, "FROM pg_catalog.pg_publication_tables t\n"
" WHERE t.pubname IN (");
get_publications_str(publications, &cmd, true);
appendStringInfoChar(&cmd, ')');
- res = walrcv_exec(wrconn, cmd.data, 2, tableRow);
+ res = walrcv_exec(wrconn, cmd.data, check_columnlist ? 3 : 2, tableRow);
pfree(cmd.data);
if (res->status != WALRCV_OK_TUPLES)
@@ -1795,7 +1802,18 @@ fetch_table_list(WalReceiverConn *wrconn, List *publications)
Assert(!isnull);
rv = makeRangeVar(nspname, relname, -1);
- tablelist = lappend(tablelist, rv);
+
+ /*
+ * We don't support the case where column list is different for the
+ * same table in different publications.
+ */
+ if (check_columnlist && list_member(tablelist, rv))
+ ereport(ERROR,
+ errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot use different column lists for table \"%s.%s\" in different publications",
+ nspname, relname));
+ else
+ tablelist = lappend(tablelist, rv);
ExecClearTuple(slot);
}
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 994c7a0..e4633bd 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -753,17 +753,6 @@ fetch_remote_table_info(char *nspname, char *relname,
/*
* Get column lists for each relation.
*
- * For initial synchronization, column lists can be ignored in following
- * cases:
- *
- * 1) one of the subscribed publications for the table hasn't specified
- * any column list
- *
- * 2) one of the subscribed publications has puballtables set to true
- *
- * 3) one of the subscribed publications is declared as ALL TABLES IN
- * SCHEMA that includes this relation
- *
* We need to do this before fetching info about column names and types,
* so that we can skip columns that should not be replicated.
*/
@@ -771,7 +760,7 @@ fetch_remote_table_info(char *nspname, char *relname,
{
WalRcvExecResult *pubres;
TupleTableSlot *slot;
- Oid attrsRow[] = {INT2OID};
+ Oid attrsRow[] = {INT2VECTOROID};
StringInfoData pub_names;
bool first = true;
@@ -786,19 +775,17 @@ fetch_remote_table_info(char *nspname, char *relname,
/*
* Fetch info about column lists for the relation (from all the
- * publications). We unnest the int2vector values, because that makes
- * it easier to combine lists by simply adding the attnums to a new
- * bitmap (without having to parse the int2vector data). This
- * preserves NULL values, so that if one of the publications has no
- * column list, we'll know that.
+ * publications).
*/
resetStringInfo(&cmd);
appendStringInfo(&cmd,
- "SELECT DISTINCT unnest"
+ "SELECT DISTINCT"
+ " (CASE WHEN (array_length(gpt.attrs, 1) = c.relnatts)"
+ " THEN NULL ELSE gpt.attrs END)"
" FROM pg_publication p,"
- " LATERAL pg_get_publication_tables(p.pubname) gpt"
- " LEFT OUTER JOIN unnest(gpt.attrs) ON TRUE"
- " WHERE gpt.relid = %u"
+ " LATERAL pg_get_publication_tables(p.pubname) gpt,"
+ " pg_class c"
+ " WHERE gpt.relid = %u AND c.oid = gpt.relid"
" AND p.pubname IN ( %s )",
lrel->remoteid,
pub_names.data);
@@ -813,26 +800,43 @@ fetch_remote_table_info(char *nspname, char *relname,
nspname, relname, pubres->err)));
/*
- * Merge the column lists (from different publications) by creating a
- * single bitmap with all the attnums. If we find a NULL value, that
- * means one of the publications has no column list for the table
- * we're syncing.
+ * We don't support the case where the column list is different for the
+ * same table when combining publications. So there should be only one
+ * row returned. Although we already checked this when creating
+ * subscription, we still need to check here in case the column list
+ * was changed after creating the subscription and before the sync
+ * worker is started.
+ */
+ if (tuplestore_tuple_count(pubres->tuplestore) > 1)
+ ereport(ERROR,
+ errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot use different column lists for table \"%s.%s\" in different publications",
+ nspname, relname));
+
+ /*
+ * Get the column list and build a single bitmap with the attnums.
+ *
+ * If we find a NULL value, it means all the columns should be
+ * replicated.
*/
slot = MakeSingleTupleTableSlot(pubres->tupledesc, &TTSOpsMinimalTuple);
- while (tuplestore_gettupleslot(pubres->tuplestore, true, false, slot))
+ if (tuplestore_gettupleslot(pubres->tuplestore, true, false, slot))
{
Datum cfval = slot_getattr(slot, 1, &isnull);
- /* NULL means empty column list, so we're done. */
- if (isnull)
+ if (!isnull)
{
- bms_free(included_cols);
- included_cols = NULL;
- break;
- }
+ ArrayType *arr;
+ int nelems;
+ int16 *elems;
- included_cols = bms_add_member(included_cols,
- DatumGetInt16(cfval));
+ arr = DatumGetArrayTypeP(cfval);
+ nelems = ARR_DIMS(arr)[0];
+ elems = (int16 *) ARR_DATA_PTR(arr);
+
+ for (natt = 0; natt < nelems; natt++)
+ included_cols = bms_add_member(included_cols, elems[natt]);
+ }
ExecClearTuple(slot);
}
diff --git a/src/backend/replication/pgoutput/pgoutput.c b/src/backend/replication/pgoutput/pgoutput.c
index 42c06af..e9079bd 100644
--- a/src/backend/replication/pgoutput/pgoutput.c
+++ b/src/backend/replication/pgoutput/pgoutput.c
@@ -979,30 +979,31 @@ pgoutput_column_list_init(PGOutputData *data, List *publications,
RelationSyncEntry *entry)
{
ListCell *lc;
+ bool first = true;
+ Relation relation = RelationIdGetRelation(entry->publish_as_relid);
/*
* Find if there are any column lists for this relation. If there are,
- * build a bitmap merging all the column lists.
- *
- * All the given publication-table mappings must be checked.
+ * build a bitmap using the column lists.
*
* Multiple publications might have multiple column lists for this
* relation.
*
+ * Note that we don't support the case where the column list is different
+ * for the same table when combining publications. But one can later change
+ * the publication so we still need to check all the given
+ * publication-table mappings and report an error if any publications have
+ * a different column list.
+ *
* FOR ALL TABLES and FOR ALL TABLES IN SCHEMA implies "don't use column
- * list" so it takes precedence.
+ * list".
*/
foreach(lc, publications)
{
Publication *pub = lfirst(lc);
HeapTuple cftuple = NULL;
Datum cfdatum = 0;
-
- /*
- * Assume there's no column list. Only if we find pg_publication_rel
- * entry with a column list we'll switch it to false.
- */
- bool pub_no_list = true;
+ Bitmapset *cols = NULL;
/*
* If the publication is FOR ALL TABLES then it is treated the same as
@@ -1011,6 +1012,8 @@ pgoutput_column_list_init(PGOutputData *data, List *publications,
*/
if (!pub->alltables)
{
+ bool pub_no_list = true;
+
/*
* Check for the presence of a column list in this publication.
*
@@ -1024,51 +1027,48 @@ pgoutput_column_list_init(PGOutputData *data, List *publications,
if (HeapTupleIsValid(cftuple))
{
- /*
- * Lookup the column list attribute.
- *
- * Note: We update the pub_no_list value directly, because if
- * the value is NULL, we have no list (and vice versa).
- */
+ /* Lookup the column list attribute. */
cfdatum = SysCacheGetAttr(PUBLICATIONRELMAP, cftuple,
Anum_pg_publication_rel_prattrs,
&pub_no_list);
- /*
- * Build the column list bitmap in the per-entry context.
- *
- * We need to merge column lists from all publications, so we
- * update the same bitmapset. If the column list is null, we
- * interpret it as replicating all columns.
- */
+ /* Build the column list bitmap in the per-entry context. */
if (!pub_no_list) /* when not null */
{
pgoutput_ensure_entry_cxt(data, entry);
- entry->columns = pub_collist_to_bitmapset(entry->columns,
- cfdatum,
- entry->entry_cxt);
+ cols = pub_collist_to_bitmapset(cols, cfdatum,
+ entry->entry_cxt);
+
+ /*
+ * If column list includes all the columns of the table,
+ * set it to NULL.
+ */
+ if (bms_num_members(cols) == RelationGetNumberOfAttributes(relation))
+ {
+ bms_free(cols);
+ cols = NULL;
+ }
}
+
+ ReleaseSysCache(cftuple);
}
}
- /*
- * Found a publication with no column list, so we're done. But first
- * discard column list we might have from preceding publications.
- */
- if (pub_no_list)
+ if (first)
{
- if (cftuple)
- ReleaseSysCache(cftuple);
-
- bms_free(entry->columns);
- entry->columns = NULL;
-
- break;
+ entry->columns = cols;
+ first = false;
}
-
- ReleaseSysCache(cftuple);
+ else if (!bms_equal(entry->columns, cols))
+ ereport(ERROR,
+ errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot use different column lists for table \"%s.%s\" in different publications",
+ get_namespace_name(RelationGetNamespace(relation)),
+ RelationGetRelationName(relation)));
} /* loop all subscribed publications */
+
+ RelationClose(relation);
}
/*
diff --git a/src/test/subscription/t/031_column_list.pl b/src/test/subscription/t/031_column_list.pl
index 19812e1..7f031bc 100644
--- a/src/test/subscription/t/031_column_list.pl
+++ b/src/test/subscription/t/031_column_list.pl
@@ -20,6 +20,7 @@ $node_subscriber->append_conf('postgresql.conf',
$node_subscriber->start;
my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres';
+my $offset = 0;
sub wait_for_subscription_sync
{
@@ -361,13 +362,13 @@ is( $result, qq(1|abc
2|xyz), 'update on column tab2.c is not replicated');
-# TEST: add a table to two publications with different column lists, and
+# TEST: add a table to two publications with same column lists, and
# create a single subscription replicating both publications
$node_publisher->safe_psql(
'postgres', qq(
CREATE TABLE tab5 (a int PRIMARY KEY, b int, c int, d int);
CREATE PUBLICATION pub2 FOR TABLE tab5 (a, b);
- CREATE PUBLICATION pub3 FOR TABLE tab5 (a, d);
+ CREATE PUBLICATION pub3 FOR TABLE tab5 (a, b);
-- insert a couple initial records
INSERT INTO tab5 VALUES (1, 11, 111, 1111);
@@ -388,8 +389,7 @@ wait_for_subscription_sync($node_subscriber);
$node_publisher->wait_for_catchup('sub1');
-# insert data and make sure all the columns (union of the columns lists)
-# get fully replicated
+# insert data and make sure the columns in column list get fully replicated
$node_publisher->safe_psql(
'postgres', qq(
INSERT INTO tab5 VALUES (3, 33, 333, 3333);
@@ -399,42 +399,12 @@ $node_publisher->safe_psql(
$node_publisher->wait_for_catchup('sub1');
is( $node_subscriber->safe_psql('postgres', "SELECT * FROM tab5 ORDER BY a"),
- qq(1|11|1111
-2|22|2222
-3|33|3333
-4|44|4444),
+ qq(1|11|
+2|22|
+3|33|
+4|44|),
'overlapping publications with overlapping column lists');
-# and finally, remove the column list for one of the publications, which
-# means replicating all columns (removing the column list), but first add
-# the missing column to the table on subscriber
-$node_publisher->safe_psql(
- 'postgres', qq(
- ALTER PUBLICATION pub3 SET TABLE tab5;
-));
-
-$node_subscriber->safe_psql(
- 'postgres', qq(
- ALTER SUBSCRIPTION sub1 REFRESH PUBLICATION;
- ALTER TABLE tab5 ADD COLUMN c INT;
-));
-
-wait_for_subscription_sync($node_subscriber);
-
-$node_publisher->safe_psql(
- 'postgres', qq(
- INSERT INTO tab5 VALUES (5, 55, 555, 5555);
-));
-
-$node_publisher->wait_for_catchup('sub1');
-
-is( $node_subscriber->safe_psql('postgres', "SELECT * FROM tab5 ORDER BY a"),
- qq(1|11|1111|
-2|22|2222|
-3|33|3333|
-4|44|4444|
-5|55|5555|555),
- 'overlapping publications with overlapping column lists');
# TEST: create a table with a column list, then change the replica
# identity by replacing a primary key (but use a different column in
@@ -900,57 +870,21 @@ is( $node_subscriber->safe_psql(
3|),
'partitions with different replica identities not replicated correctly');
-# TEST: With a table included in multiple publications, we should use a
-# union of the column lists. So with column lists (a,b) and (a,c) we
-# should replicate (a,b,c).
-
-$node_publisher->safe_psql(
- 'postgres', qq(
- CREATE TABLE test_mix_1 (a int PRIMARY KEY, b int, c int);
- CREATE PUBLICATION pub_mix_1 FOR TABLE test_mix_1 (a, b);
- CREATE PUBLICATION pub_mix_2 FOR TABLE test_mix_1 (a, c);
-
- -- initial data
- INSERT INTO test_mix_1 VALUES (1, 2, 3);
-));
-
-$node_subscriber->safe_psql(
- 'postgres', qq(
- CREATE TABLE test_mix_1 (a int PRIMARY KEY, b int, c int);
- ALTER SUBSCRIPTION sub1 SET PUBLICATION pub_mix_1, pub_mix_2;
-));
-
-wait_for_subscription_sync($node_subscriber);
-
-$node_publisher->safe_psql(
- 'postgres', qq(
- INSERT INTO test_mix_1 VALUES (4, 5, 6);
-));
-
-$node_publisher->wait_for_catchup('sub1');
-
-is( $node_subscriber->safe_psql(
- 'postgres', "SELECT * FROM test_mix_1 ORDER BY a"),
- qq(1|2|3
-4|5|6),
- 'a mix of publications should use a union of column list');
-
-# TEST: With a table included in multiple publications, we should use a
-# union of the column lists. If any of the publications is FOR ALL
-# TABLES, we should replicate all columns.
+# TEST: With a table included in the publications which is FOR ALL TABLES, it
+# means replicate all columns.
# drop unnecessary tables, so as not to interfere with the FOR ALL TABLES
$node_publisher->safe_psql(
'postgres', qq(
- DROP TABLE tab1, tab2, tab3, tab4, tab5, tab6, tab7, test_mix_1,
+ DROP TABLE tab1, tab2, tab3, tab4, tab5, tab6, tab7,
test_part, test_part_a, test_part_b, test_part_c, test_part_d;
));
$node_publisher->safe_psql(
'postgres', qq(
CREATE TABLE test_mix_2 (a int PRIMARY KEY, b int, c int);
- CREATE PUBLICATION pub_mix_3 FOR TABLE test_mix_2 (a, b);
+ CREATE PUBLICATION pub_mix_3 FOR TABLE test_mix_2 (a, b, c);
CREATE PUBLICATION pub_mix_4 FOR ALL TABLES;
-- initial data
@@ -976,12 +910,11 @@ $node_publisher->wait_for_catchup('sub1');
is( $node_subscriber->safe_psql('postgres', "SELECT * FROM test_mix_2"),
qq(1|2|3
4|5|6),
- 'a mix of publications should use a union of column list');
+ 'all columns should be replicated');
-# TEST: With a table included in multiple publications, we should use a
-# union of the column lists. If any of the publications is FOR ALL
-# TABLES IN SCHEMA, we should replicate all columns.
+# TEST: With a table included in the publication which is FOR ALL TABLES
+# IN SCHEMA, it means replicate all columns.
$node_subscriber->safe_psql(
'postgres', qq(
@@ -993,7 +926,7 @@ $node_publisher->safe_psql(
'postgres', qq(
DROP TABLE test_mix_2;
CREATE TABLE test_mix_3 (a int PRIMARY KEY, b int, c int);
- CREATE PUBLICATION pub_mix_5 FOR TABLE test_mix_3 (a, b);
+ CREATE PUBLICATION pub_mix_5 FOR TABLE test_mix_3 (a, b, c);
CREATE PUBLICATION pub_mix_6 FOR ALL TABLES IN SCHEMA public;
-- initial data
@@ -1017,7 +950,7 @@ $node_publisher->wait_for_catchup('sub1');
is( $node_subscriber->safe_psql('postgres', "SELECT * FROM test_mix_3"),
qq(1|2|3
4|5|6),
- 'a mix of publications should use a union of column list');
+ 'all columns should be replicated');
# TEST: Check handling of publish_via_partition_root - if a partition is
@@ -1074,7 +1007,7 @@ is( $node_subscriber->safe_psql(
# TEST: Multiple publications which publish schema of parent table and
# partition. The partition is published through two publications, once
# through a schema (so no column list) containing the parent, and then
-# also directly (with a columns list). The expected outcome is there is
+# also directly (with all columns). The expected outcome is there is
# no column list.
$node_publisher->safe_psql(
@@ -1086,7 +1019,7 @@ $node_publisher->safe_psql(
CREATE TABLE t_1 PARTITION OF s1.t FOR VALUES FROM (1) TO (10);
CREATE PUBLICATION pub1 FOR ALL TABLES IN SCHEMA s1;
- CREATE PUBLICATION pub2 FOR TABLE t_1(b);
+ CREATE PUBLICATION pub2 FOR TABLE t_1(a, b, c);
-- initial data
INSERT INTO s1.t VALUES (1, 2, 3);
@@ -1233,6 +1166,51 @@ is( $node_subscriber->safe_psql(
'publication containing both parent and child relation');
+# TEST: With a table included in multiple publications with different column
+# lists, we should catch the error when creating the subscription.
+
+$node_publisher->safe_psql('postgres', qq(
+ CREATE TABLE test_mix_1 (a int PRIMARY KEY, b int, c int);
+ CREATE PUBLICATION pub_mix_1 FOR TABLE test_mix_1 (a, b);
+ CREATE PUBLICATION pub_mix_2 FOR TABLE test_mix_1 (a, c);
+));
+
+$node_subscriber->safe_psql('postgres', qq(
+ DROP SUBSCRIPTION sub1;
+ CREATE TABLE test_mix_1 (a int PRIMARY KEY, b int, c int);
+));
+
+my ($cmdret, $stdout, $stderr) = $node_subscriber->psql(
+ 'postgres', qq(
+ CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_mix_1, pub_mix_2;
+));
+
+ok( $stderr =~
+ qr/cannot use different column lists for table "public.test_mix_1" in different publications/,
+ 'different column lists detected');
+
+# TEST: If the column list is changed after creating the subscription, we
+# should catch the error reported by walsender.
+
+$node_publisher->safe_psql('postgres', qq(
+ ALTER PUBLICATION pub_mix_1 SET TABLE test_mix_1 (a, c);
+));
+
+$node_subscriber->safe_psql('postgres', qq(
+ CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_mix_1, pub_mix_2;
+));
+
+$node_publisher->wait_for_catchup('sub1');
+
+$node_publisher->safe_psql('postgres', qq(
+ ALTER PUBLICATION pub_mix_1 SET TABLE test_mix_1 (a, b);
+ INSERT INTO test_mix_1 VALUES(1, 1, 1);
+));
+
+$offset = $node_publisher->wait_for_log(
+ qr/cannot use different column lists for table "public.test_mix_1" in different publications/,
+ $offset);
+
$node_subscriber->stop('fast');
$node_publisher->stop('fast');
--
2.7.2.windows.1
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-24 09:49 Amit Kapila <[email protected]>
parent: Amit Kapila <[email protected]>
1 sibling, 1 reply; 58+ messages in thread
From: Amit Kapila @ 2022-05-24 09:49 UTC (permalink / raw)
To: Tom Lane <[email protected]>; Alvaro Herrera <[email protected]>; +Cc: Justin Pryzby <[email protected]>; Hou, Zhijie/侯 志杰 <[email protected]>; Peter Eisentraut <[email protected]>; PostgreSQL Hackers <[email protected]>; Tomas Vondra <[email protected]>
On Fri, May 20, 2022 at 8:36 AM Amit Kapila <[email protected]> wrote:
>
> On Thu, May 19, 2022 at 7:54 PM Tom Lane <[email protected]> wrote:
> >
> > Justin Pryzby <[email protected]> writes:
> > > On Thu, May 19, 2022 at 10:33:13AM +0530, Amit Kapila wrote:
> > >> I have committed the first patch after fixing this part. It seems Tom
> > >> is not very happy doing this after beta-1 [1]. The reason we get this
> > >> information via this view (and underlying function) is that it
> > >> simplifies the queries on the subscriber-side as you can see in the
> > >> second patch. The query change is as below:
> > >> [1] - https://www.postgresql.org/message-id/91075.1652929852%40sss.pgh.pa.us
> >
> > > I think Tom's concern is that adding information to a view seems like adding a
> > > feature that hadn't previously been contemplated.
> > > (Catalog changes themselves are not prohibited during the beta period).
> >
> > It certainly smells like a new feature, but my concern was more around the
> > post-beta catalog change. We do those only if really forced to, and the
> > explanation in the commit message didn't satisfy me as to why it was
> > necessary. This explanation isn't much better --- if we're trying to
> > prohibit a certain class of publication definitions, what good does it do
> > to check that on the subscriber side?
> >
>
> It is required on the subscriber side because prohibition is only for
> the cases where multiple publications are combined. We disallow the
> cases where the column list is different for the same table when
> combining publications. For example:
>
> Publisher-side:
> Create table tab(c1 int, c2 int);
> Create Publication pub1 for table tab(c1);
> Create Publication pub1 for table tab(c2);
>
> Subscriber-side:
> Create Subscription sub1 Connection 'dbname=postgres' Publication pub1, pub2;
>
> We want to prohibit such cases. So, it would be better to check at the
> time of 'Create Subscription' to validate such combinations and
> prohibit them. To achieve that we extended the existing function
> pg_get_publication_tables() and view pg_publication_tables to expose
> the column list and verify such a combination. We primarily need
> column list information for this prohibition but it appeared natural
> to expose the row filter.
>
I still feel that the current approach to extend the underlying
function and view is a better idea but if you and or others are not
convinced then we can try to achieve it by extending the existing
query on the subscriber side as mentioned in my previous email [1].
Kindly let me know your opinion?
[1] - https://www.postgresql.org/message-id/CAA4eK1KfL%3Dez5fKPB-0Nrgf7wiqN9bXP-YHHj2YH5utXAmjYug%40mail.g...
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-26 03:26 Amit Kapila <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 0 replies; 58+ messages in thread
From: Amit Kapila @ 2022-05-26 03:26 UTC (permalink / raw)
To: Tom Lane <[email protected]>; Alvaro Herrera <[email protected]>; +Cc: Justin Pryzby <[email protected]>; Hou, Zhijie/侯 志杰 <[email protected]>; Peter Eisentraut <[email protected]>; PostgreSQL Hackers <[email protected]>; Tomas Vondra <[email protected]>
On Tue, May 24, 2022 at 3:19 PM Amit Kapila <[email protected]> wrote:
>
> On Fri, May 20, 2022 at 8:36 AM Amit Kapila <[email protected]> wrote:
> >
> > On Thu, May 19, 2022 at 7:54 PM Tom Lane <[email protected]> wrote:
> > >
> > > Justin Pryzby <[email protected]> writes:
> > > > On Thu, May 19, 2022 at 10:33:13AM +0530, Amit Kapila wrote:
> > > >> I have committed the first patch after fixing this part. It seems Tom
> > > >> is not very happy doing this after beta-1 [1]. The reason we get this
> > > >> information via this view (and underlying function) is that it
> > > >> simplifies the queries on the subscriber-side as you can see in the
> > > >> second patch. The query change is as below:
> > > >> [1] - https://www.postgresql.org/message-id/91075.1652929852%40sss.pgh.pa.us
> > >
> > > > I think Tom's concern is that adding information to a view seems like adding a
> > > > feature that hadn't previously been contemplated.
> > > > (Catalog changes themselves are not prohibited during the beta period).
> > >
> > > It certainly smells like a new feature, but my concern was more around the
> > > post-beta catalog change. We do those only if really forced to, and the
> > > explanation in the commit message didn't satisfy me as to why it was
> > > necessary. This explanation isn't much better --- if we're trying to
> > > prohibit a certain class of publication definitions, what good does it do
> > > to check that on the subscriber side?
> > >
> >
> > It is required on the subscriber side because prohibition is only for
> > the cases where multiple publications are combined. We disallow the
> > cases where the column list is different for the same table when
> > combining publications. For example:
> >
> > Publisher-side:
> > Create table tab(c1 int, c2 int);
> > Create Publication pub1 for table tab(c1);
> > Create Publication pub1 for table tab(c2);
> >
> > Subscriber-side:
> > Create Subscription sub1 Connection 'dbname=postgres' Publication pub1, pub2;
> >
> > We want to prohibit such cases. So, it would be better to check at the
> > time of 'Create Subscription' to validate such combinations and
> > prohibit them. To achieve that we extended the existing function
> > pg_get_publication_tables() and view pg_publication_tables to expose
> > the column list and verify such a combination. We primarily need
> > column list information for this prohibition but it appeared natural
> > to expose the row filter.
> >
>
> I still feel that the current approach to extend the underlying
> function and view is a better idea but if you and or others are not
> convinced then we can try to achieve it by extending the existing
> query on the subscriber side as mentioned in my previous email [1].
> Kindly let me know your opinion?
>
Unless someone has objections or thinks otherwise, I am planning to
proceed with the approach of extending the function/view (patch for
which is already committed) and using it to prohibit the combinations
of publications having different column lists as is done in the
currently proposed patch [1].
[1] - https://www.postgresql.org/message-id/OS0PR01MB5716AD7C0FE7386630BDBAAB94D79%40OS0PR01MB5716.jpnprd0...
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-27 05:47 Amit Kapila <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 1 reply; 58+ messages in thread
From: Amit Kapila @ 2022-05-27 05:47 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Justin Pryzby <[email protected]>; Alvaro Herrera <[email protected]>; Tomas Vondra <[email protected]>; Peter Eisentraut <[email protected]>; PostgreSQL Hackers <[email protected]>; Tom Lane <[email protected]>
On Tue, May 24, 2022 at 11:03 AM [email protected]
<[email protected]> wrote:
>
> On Friday, May 20, 2022 11:06 AM Amit Kapila <[email protected]> wrote:
>
> Thanks for pointing it out. Here is the new version patch which add this version check.
>
I have added/edited a few comments and ran pgindent. The attached
looks good to me. I'll push this early next week unless there are more
comments/suggestions.
--
With Regards,
Amit Kapila.
Attachments:
[application/octet-stream] v5-0001-Prohibit-combining-publications-with-different-co.patch (22.4K, ../../CAA4eK1JDuOb96naN7akLpXz6G2ZGnGyg73w5Xt8ncAhWNF5Giw@mail.gmail.com/2-v5-0001-Prohibit-combining-publications-with-different-co.patch)
download | inline diff:
From 201721776782809c3bb99bb6ff8fbdedb02483ab Mon Sep 17 00:00:00 2001
From: Amit Kapila <[email protected]>
Date: Fri, 27 May 2022 08:30:33 +0530
Subject: [PATCH v5] Prohibit combining publications with different column
lists.
The main purpose of introducing a column list is to have statically
different shapes on publisher and subscriber or hide sensitive column
data. In both cases, it doesn't seem to make sense to combine column
lists.
So, we disallow the cases where the column list is different for the same
table when combining publications. It can be later extended to combine the
column lists for selective cases where required.
Reported-by: Alvaro Herrera
Author: Hou Zhijie
Reviewed-by: Amit Kapila, Alvaro Herrera, Shi Yu, Takamichi Osumi
Discussion: https://postgr.es/m/[email protected]
---
doc/src/sgml/ref/alter_publication.sgml | 12 ++-
doc/src/sgml/ref/create_subscription.sgml | 5 +
src/backend/commands/subscriptioncmds.c | 28 +++++-
src/backend/replication/logical/tablesync.c | 72 ++++++-------
src/backend/replication/pgoutput/pgoutput.c | 80 +++++++--------
src/test/subscription/t/031_column_list.pl | 150 ++++++++++++----------------
6 files changed, 181 insertions(+), 166 deletions(-)
diff --git a/doc/src/sgml/ref/alter_publication.sgml b/doc/src/sgml/ref/alter_publication.sgml
index e2cce49..f03933a 100644
--- a/doc/src/sgml/ref/alter_publication.sgml
+++ b/doc/src/sgml/ref/alter_publication.sgml
@@ -116,7 +116,17 @@ ALTER PUBLICATION <replaceable class="parameter">name</replaceable> RENAME TO <r
<para>
Optionally, a column list can be specified. See <xref
- linkend="sql-createpublication"/> for details.
+ linkend="sql-createpublication"/> for details. Note that a subscription
+ having several publications in which the same table has been published
+ with different column lists is not supported. So, changing the column
+ lists of the tables being subscribed could cause inconsistency of column
+ lists among publications in which case <command>ALTER PUBLICATION</command>
+ command will be successful but later the WalSender in publisher or the
+ subscriber may throw an error. In this scenario, the user needs to
+ recreate the subscription after adjusting the column list or drop the
+ problematic publication using
+ <literal>ALTER SUBSCRIPTION ... DROP PUBLICATION</literal> and then add
+ it back after adjusting the column list.
</para>
<para>
diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml
index 203bb41..f6f82a0 100644
--- a/doc/src/sgml/ref/create_subscription.sgml
+++ b/doc/src/sgml/ref/create_subscription.sgml
@@ -356,6 +356,11 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl
</para>
<para>
+ Subscription having several publications in which the same table has been
+ published with different column lists is not supported.
+ </para>
+
+ <para>
We allow non-existent publications to be specified so that users can add
those later. This means
<link linkend="catalog-pg-subscription"><structname>pg_subscription</structname></link>
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 690cdaa..83e6eae 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1754,6 +1754,11 @@ AlterSubscriptionOwner_oid(Oid subid, Oid newOwnerId)
/*
* Get the list of tables which belong to specified publications on the
* publisher connection.
+ *
+ * Note that we don't support the case where the column list is different for
+ * the same table in different publications to avoid sending unwanted column
+ * information for some of the rows. This can happen when both the column
+ * list and row filter are specified for different publications.
*/
static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
@@ -1761,17 +1766,23 @@ fetch_table_list(WalReceiverConn *wrconn, List *publications)
WalRcvExecResult *res;
StringInfoData cmd;
TupleTableSlot *slot;
- Oid tableRow[2] = {TEXTOID, TEXTOID};
+ Oid tableRow[3] = {TEXTOID, TEXTOID, NAMEARRAYOID};
List *tablelist = NIL;
+ bool check_columnlist = (walrcv_server_version(wrconn) >= 150000);
initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename\n"
- " FROM pg_catalog.pg_publication_tables t\n"
+ appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename \n");
+
+ /* Get column lists for each relation if the publisher supports it */
+ if (check_columnlist)
+ appendStringInfoString(&cmd, ", t.attnames\n");
+
+ appendStringInfoString(&cmd, "FROM pg_catalog.pg_publication_tables t\n"
" WHERE t.pubname IN (");
get_publications_str(publications, &cmd, true);
appendStringInfoChar(&cmd, ')');
- res = walrcv_exec(wrconn, cmd.data, 2, tableRow);
+ res = walrcv_exec(wrconn, cmd.data, check_columnlist ? 3 : 2, tableRow);
pfree(cmd.data);
if (res->status != WALRCV_OK_TUPLES)
@@ -1795,7 +1806,14 @@ fetch_table_list(WalReceiverConn *wrconn, List *publications)
Assert(!isnull);
rv = makeRangeVar(nspname, relname, -1);
- tablelist = lappend(tablelist, rv);
+
+ if (check_columnlist && list_member(tablelist, rv))
+ ereport(ERROR,
+ errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot use different column lists for table \"%s.%s\" in different publications",
+ nspname, relname));
+ else
+ tablelist = lappend(tablelist, rv);
ExecClearTuple(slot);
}
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 994c7a0..670c6fc 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -753,17 +753,6 @@ fetch_remote_table_info(char *nspname, char *relname,
/*
* Get column lists for each relation.
*
- * For initial synchronization, column lists can be ignored in following
- * cases:
- *
- * 1) one of the subscribed publications for the table hasn't specified
- * any column list
- *
- * 2) one of the subscribed publications has puballtables set to true
- *
- * 3) one of the subscribed publications is declared as ALL TABLES IN
- * SCHEMA that includes this relation
- *
* We need to do this before fetching info about column names and types,
* so that we can skip columns that should not be replicated.
*/
@@ -771,7 +760,7 @@ fetch_remote_table_info(char *nspname, char *relname,
{
WalRcvExecResult *pubres;
TupleTableSlot *slot;
- Oid attrsRow[] = {INT2OID};
+ Oid attrsRow[] = {INT2VECTOROID};
StringInfoData pub_names;
bool first = true;
@@ -786,19 +775,17 @@ fetch_remote_table_info(char *nspname, char *relname,
/*
* Fetch info about column lists for the relation (from all the
- * publications). We unnest the int2vector values, because that makes
- * it easier to combine lists by simply adding the attnums to a new
- * bitmap (without having to parse the int2vector data). This
- * preserves NULL values, so that if one of the publications has no
- * column list, we'll know that.
+ * publications).
*/
resetStringInfo(&cmd);
appendStringInfo(&cmd,
- "SELECT DISTINCT unnest"
+ "SELECT DISTINCT"
+ " (CASE WHEN (array_length(gpt.attrs, 1) = c.relnatts)"
+ " THEN NULL ELSE gpt.attrs END)"
" FROM pg_publication p,"
- " LATERAL pg_get_publication_tables(p.pubname) gpt"
- " LEFT OUTER JOIN unnest(gpt.attrs) ON TRUE"
- " WHERE gpt.relid = %u"
+ " LATERAL pg_get_publication_tables(p.pubname) gpt,"
+ " pg_class c"
+ " WHERE gpt.relid = %u AND c.oid = gpt.relid"
" AND p.pubname IN ( %s )",
lrel->remoteid,
pub_names.data);
@@ -813,26 +800,43 @@ fetch_remote_table_info(char *nspname, char *relname,
nspname, relname, pubres->err)));
/*
- * Merge the column lists (from different publications) by creating a
- * single bitmap with all the attnums. If we find a NULL value, that
- * means one of the publications has no column list for the table
- * we're syncing.
+ * We don't support the case where the column list is different for
+ * the same table when combining publications. See comments atop
+ * fetch_table_list. So there should be only one row returned.
+ * Although we already checked this when creating the subscription, we
+ * still need to check here in case the column list was changed after
+ * creating the subscription and before the sync worker is started.
+ */
+ if (tuplestore_tuple_count(pubres->tuplestore) > 1)
+ ereport(ERROR,
+ errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot use different column lists for table \"%s.%s\" in different publications",
+ nspname, relname));
+
+ /*
+ * Get the column list and build a single bitmap with the attnums.
+ *
+ * If we find a NULL value, it means all the columns should be
+ * replicated.
*/
slot = MakeSingleTupleTableSlot(pubres->tupledesc, &TTSOpsMinimalTuple);
- while (tuplestore_gettupleslot(pubres->tuplestore, true, false, slot))
+ if (tuplestore_gettupleslot(pubres->tuplestore, true, false, slot))
{
Datum cfval = slot_getattr(slot, 1, &isnull);
- /* NULL means empty column list, so we're done. */
- if (isnull)
+ if (!isnull)
{
- bms_free(included_cols);
- included_cols = NULL;
- break;
- }
+ ArrayType *arr;
+ int nelems;
+ int16 *elems;
- included_cols = bms_add_member(included_cols,
- DatumGetInt16(cfval));
+ arr = DatumGetArrayTypeP(cfval);
+ nelems = ARR_DIMS(arr)[0];
+ elems = (int16 *) ARR_DATA_PTR(arr);
+
+ for (natt = 0; natt < nelems; natt++)
+ included_cols = bms_add_member(included_cols, elems[natt]);
+ }
ExecClearTuple(slot);
}
diff --git a/src/backend/replication/pgoutput/pgoutput.c b/src/backend/replication/pgoutput/pgoutput.c
index 42c06af..8deae57 100644
--- a/src/backend/replication/pgoutput/pgoutput.c
+++ b/src/backend/replication/pgoutput/pgoutput.c
@@ -979,30 +979,31 @@ pgoutput_column_list_init(PGOutputData *data, List *publications,
RelationSyncEntry *entry)
{
ListCell *lc;
+ bool first = true;
+ Relation relation = RelationIdGetRelation(entry->publish_as_relid);
/*
* Find if there are any column lists for this relation. If there are,
- * build a bitmap merging all the column lists.
- *
- * All the given publication-table mappings must be checked.
+ * build a bitmap using the column lists.
*
* Multiple publications might have multiple column lists for this
* relation.
*
+ * Note that we don't support the case where the column list is different
+ * for the same table when combining publications. See comments atop
+ * fetch_table_list. But one can later change the publication so we still
+ * need to check all the given publication-table mappings and report an
+ * error if any publications have a different column list.
+ *
* FOR ALL TABLES and FOR ALL TABLES IN SCHEMA implies "don't use column
- * list" so it takes precedence.
+ * list".
*/
foreach(lc, publications)
{
Publication *pub = lfirst(lc);
HeapTuple cftuple = NULL;
Datum cfdatum = 0;
-
- /*
- * Assume there's no column list. Only if we find pg_publication_rel
- * entry with a column list we'll switch it to false.
- */
- bool pub_no_list = true;
+ Bitmapset *cols = NULL;
/*
* If the publication is FOR ALL TABLES then it is treated the same as
@@ -1011,6 +1012,8 @@ pgoutput_column_list_init(PGOutputData *data, List *publications,
*/
if (!pub->alltables)
{
+ bool pub_no_list = true;
+
/*
* Check for the presence of a column list in this publication.
*
@@ -1024,51 +1027,48 @@ pgoutput_column_list_init(PGOutputData *data, List *publications,
if (HeapTupleIsValid(cftuple))
{
- /*
- * Lookup the column list attribute.
- *
- * Note: We update the pub_no_list value directly, because if
- * the value is NULL, we have no list (and vice versa).
- */
+ /* Lookup the column list attribute. */
cfdatum = SysCacheGetAttr(PUBLICATIONRELMAP, cftuple,
Anum_pg_publication_rel_prattrs,
&pub_no_list);
- /*
- * Build the column list bitmap in the per-entry context.
- *
- * We need to merge column lists from all publications, so we
- * update the same bitmapset. If the column list is null, we
- * interpret it as replicating all columns.
- */
+ /* Build the column list bitmap in the per-entry context. */
if (!pub_no_list) /* when not null */
{
pgoutput_ensure_entry_cxt(data, entry);
- entry->columns = pub_collist_to_bitmapset(entry->columns,
- cfdatum,
- entry->entry_cxt);
+ cols = pub_collist_to_bitmapset(cols, cfdatum,
+ entry->entry_cxt);
+
+ /*
+ * If column list includes all the columns of the table,
+ * set it to NULL.
+ */
+ if (bms_num_members(cols) == RelationGetNumberOfAttributes(relation))
+ {
+ bms_free(cols);
+ cols = NULL;
+ }
}
+
+ ReleaseSysCache(cftuple);
}
}
- /*
- * Found a publication with no column list, so we're done. But first
- * discard column list we might have from preceding publications.
- */
- if (pub_no_list)
+ if (first)
{
- if (cftuple)
- ReleaseSysCache(cftuple);
-
- bms_free(entry->columns);
- entry->columns = NULL;
-
- break;
+ entry->columns = cols;
+ first = false;
}
-
- ReleaseSysCache(cftuple);
+ else if (!bms_equal(entry->columns, cols))
+ ereport(ERROR,
+ errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot use different column lists for table \"%s.%s\" in different publications",
+ get_namespace_name(RelationGetNamespace(relation)),
+ RelationGetRelationName(relation)));
} /* loop all subscribed publications */
+
+ RelationClose(relation);
}
/*
diff --git a/src/test/subscription/t/031_column_list.pl b/src/test/subscription/t/031_column_list.pl
index 19812e1..7f031bc 100644
--- a/src/test/subscription/t/031_column_list.pl
+++ b/src/test/subscription/t/031_column_list.pl
@@ -20,6 +20,7 @@ $node_subscriber->append_conf('postgresql.conf',
$node_subscriber->start;
my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres';
+my $offset = 0;
sub wait_for_subscription_sync
{
@@ -361,13 +362,13 @@ is( $result, qq(1|abc
2|xyz), 'update on column tab2.c is not replicated');
-# TEST: add a table to two publications with different column lists, and
+# TEST: add a table to two publications with same column lists, and
# create a single subscription replicating both publications
$node_publisher->safe_psql(
'postgres', qq(
CREATE TABLE tab5 (a int PRIMARY KEY, b int, c int, d int);
CREATE PUBLICATION pub2 FOR TABLE tab5 (a, b);
- CREATE PUBLICATION pub3 FOR TABLE tab5 (a, d);
+ CREATE PUBLICATION pub3 FOR TABLE tab5 (a, b);
-- insert a couple initial records
INSERT INTO tab5 VALUES (1, 11, 111, 1111);
@@ -388,8 +389,7 @@ wait_for_subscription_sync($node_subscriber);
$node_publisher->wait_for_catchup('sub1');
-# insert data and make sure all the columns (union of the columns lists)
-# get fully replicated
+# insert data and make sure the columns in column list get fully replicated
$node_publisher->safe_psql(
'postgres', qq(
INSERT INTO tab5 VALUES (3, 33, 333, 3333);
@@ -399,42 +399,12 @@ $node_publisher->safe_psql(
$node_publisher->wait_for_catchup('sub1');
is( $node_subscriber->safe_psql('postgres', "SELECT * FROM tab5 ORDER BY a"),
- qq(1|11|1111
-2|22|2222
-3|33|3333
-4|44|4444),
+ qq(1|11|
+2|22|
+3|33|
+4|44|),
'overlapping publications with overlapping column lists');
-# and finally, remove the column list for one of the publications, which
-# means replicating all columns (removing the column list), but first add
-# the missing column to the table on subscriber
-$node_publisher->safe_psql(
- 'postgres', qq(
- ALTER PUBLICATION pub3 SET TABLE tab5;
-));
-
-$node_subscriber->safe_psql(
- 'postgres', qq(
- ALTER SUBSCRIPTION sub1 REFRESH PUBLICATION;
- ALTER TABLE tab5 ADD COLUMN c INT;
-));
-
-wait_for_subscription_sync($node_subscriber);
-
-$node_publisher->safe_psql(
- 'postgres', qq(
- INSERT INTO tab5 VALUES (5, 55, 555, 5555);
-));
-
-$node_publisher->wait_for_catchup('sub1');
-
-is( $node_subscriber->safe_psql('postgres', "SELECT * FROM tab5 ORDER BY a"),
- qq(1|11|1111|
-2|22|2222|
-3|33|3333|
-4|44|4444|
-5|55|5555|555),
- 'overlapping publications with overlapping column lists');
# TEST: create a table with a column list, then change the replica
# identity by replacing a primary key (but use a different column in
@@ -900,57 +870,21 @@ is( $node_subscriber->safe_psql(
3|),
'partitions with different replica identities not replicated correctly');
-# TEST: With a table included in multiple publications, we should use a
-# union of the column lists. So with column lists (a,b) and (a,c) we
-# should replicate (a,b,c).
-
-$node_publisher->safe_psql(
- 'postgres', qq(
- CREATE TABLE test_mix_1 (a int PRIMARY KEY, b int, c int);
- CREATE PUBLICATION pub_mix_1 FOR TABLE test_mix_1 (a, b);
- CREATE PUBLICATION pub_mix_2 FOR TABLE test_mix_1 (a, c);
-
- -- initial data
- INSERT INTO test_mix_1 VALUES (1, 2, 3);
-));
-
-$node_subscriber->safe_psql(
- 'postgres', qq(
- CREATE TABLE test_mix_1 (a int PRIMARY KEY, b int, c int);
- ALTER SUBSCRIPTION sub1 SET PUBLICATION pub_mix_1, pub_mix_2;
-));
-
-wait_for_subscription_sync($node_subscriber);
-
-$node_publisher->safe_psql(
- 'postgres', qq(
- INSERT INTO test_mix_1 VALUES (4, 5, 6);
-));
-
-$node_publisher->wait_for_catchup('sub1');
-
-is( $node_subscriber->safe_psql(
- 'postgres', "SELECT * FROM test_mix_1 ORDER BY a"),
- qq(1|2|3
-4|5|6),
- 'a mix of publications should use a union of column list');
-
-# TEST: With a table included in multiple publications, we should use a
-# union of the column lists. If any of the publications is FOR ALL
-# TABLES, we should replicate all columns.
+# TEST: With a table included in the publications which is FOR ALL TABLES, it
+# means replicate all columns.
# drop unnecessary tables, so as not to interfere with the FOR ALL TABLES
$node_publisher->safe_psql(
'postgres', qq(
- DROP TABLE tab1, tab2, tab3, tab4, tab5, tab6, tab7, test_mix_1,
+ DROP TABLE tab1, tab2, tab3, tab4, tab5, tab6, tab7,
test_part, test_part_a, test_part_b, test_part_c, test_part_d;
));
$node_publisher->safe_psql(
'postgres', qq(
CREATE TABLE test_mix_2 (a int PRIMARY KEY, b int, c int);
- CREATE PUBLICATION pub_mix_3 FOR TABLE test_mix_2 (a, b);
+ CREATE PUBLICATION pub_mix_3 FOR TABLE test_mix_2 (a, b, c);
CREATE PUBLICATION pub_mix_4 FOR ALL TABLES;
-- initial data
@@ -976,12 +910,11 @@ $node_publisher->wait_for_catchup('sub1');
is( $node_subscriber->safe_psql('postgres', "SELECT * FROM test_mix_2"),
qq(1|2|3
4|5|6),
- 'a mix of publications should use a union of column list');
+ 'all columns should be replicated');
-# TEST: With a table included in multiple publications, we should use a
-# union of the column lists. If any of the publications is FOR ALL
-# TABLES IN SCHEMA, we should replicate all columns.
+# TEST: With a table included in the publication which is FOR ALL TABLES
+# IN SCHEMA, it means replicate all columns.
$node_subscriber->safe_psql(
'postgres', qq(
@@ -993,7 +926,7 @@ $node_publisher->safe_psql(
'postgres', qq(
DROP TABLE test_mix_2;
CREATE TABLE test_mix_3 (a int PRIMARY KEY, b int, c int);
- CREATE PUBLICATION pub_mix_5 FOR TABLE test_mix_3 (a, b);
+ CREATE PUBLICATION pub_mix_5 FOR TABLE test_mix_3 (a, b, c);
CREATE PUBLICATION pub_mix_6 FOR ALL TABLES IN SCHEMA public;
-- initial data
@@ -1017,7 +950,7 @@ $node_publisher->wait_for_catchup('sub1');
is( $node_subscriber->safe_psql('postgres', "SELECT * FROM test_mix_3"),
qq(1|2|3
4|5|6),
- 'a mix of publications should use a union of column list');
+ 'all columns should be replicated');
# TEST: Check handling of publish_via_partition_root - if a partition is
@@ -1074,7 +1007,7 @@ is( $node_subscriber->safe_psql(
# TEST: Multiple publications which publish schema of parent table and
# partition. The partition is published through two publications, once
# through a schema (so no column list) containing the parent, and then
-# also directly (with a columns list). The expected outcome is there is
+# also directly (with all columns). The expected outcome is there is
# no column list.
$node_publisher->safe_psql(
@@ -1086,7 +1019,7 @@ $node_publisher->safe_psql(
CREATE TABLE t_1 PARTITION OF s1.t FOR VALUES FROM (1) TO (10);
CREATE PUBLICATION pub1 FOR ALL TABLES IN SCHEMA s1;
- CREATE PUBLICATION pub2 FOR TABLE t_1(b);
+ CREATE PUBLICATION pub2 FOR TABLE t_1(a, b, c);
-- initial data
INSERT INTO s1.t VALUES (1, 2, 3);
@@ -1233,6 +1166,51 @@ is( $node_subscriber->safe_psql(
'publication containing both parent and child relation');
+# TEST: With a table included in multiple publications with different column
+# lists, we should catch the error when creating the subscription.
+
+$node_publisher->safe_psql('postgres', qq(
+ CREATE TABLE test_mix_1 (a int PRIMARY KEY, b int, c int);
+ CREATE PUBLICATION pub_mix_1 FOR TABLE test_mix_1 (a, b);
+ CREATE PUBLICATION pub_mix_2 FOR TABLE test_mix_1 (a, c);
+));
+
+$node_subscriber->safe_psql('postgres', qq(
+ DROP SUBSCRIPTION sub1;
+ CREATE TABLE test_mix_1 (a int PRIMARY KEY, b int, c int);
+));
+
+my ($cmdret, $stdout, $stderr) = $node_subscriber->psql(
+ 'postgres', qq(
+ CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_mix_1, pub_mix_2;
+));
+
+ok( $stderr =~
+ qr/cannot use different column lists for table "public.test_mix_1" in different publications/,
+ 'different column lists detected');
+
+# TEST: If the column list is changed after creating the subscription, we
+# should catch the error reported by walsender.
+
+$node_publisher->safe_psql('postgres', qq(
+ ALTER PUBLICATION pub_mix_1 SET TABLE test_mix_1 (a, c);
+));
+
+$node_subscriber->safe_psql('postgres', qq(
+ CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_mix_1, pub_mix_2;
+));
+
+$node_publisher->wait_for_catchup('sub1');
+
+$node_publisher->safe_psql('postgres', qq(
+ ALTER PUBLICATION pub_mix_1 SET TABLE test_mix_1 (a, b);
+ INSERT INTO test_mix_1 VALUES(1, 1, 1);
+));
+
+$offset = $node_publisher->wait_for_log(
+ qr/cannot use different column lists for table "public.test_mix_1" in different publications/,
+ $offset);
+
$node_subscriber->stop('fast');
$node_publisher->stop('fast');
--
1.8.3.1
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-05-27 05:53 Justin Pryzby <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 1 reply; 58+ messages in thread
From: Justin Pryzby @ 2022-05-27 05:53 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: [email protected]; Alvaro Herrera <[email protected]>; Tomas Vondra <[email protected]>; Peter Eisentraut <[email protected]>; [email protected]; Tom Lane <[email protected]>
On Fri, May 27, 2022 at 11:17:00AM +0530, Amit Kapila wrote:
> On Tue, May 24, 2022 at 11:03 AM [email protected] <[email protected]> wrote:
> >
> > On Friday, May 20, 2022 11:06 AM Amit Kapila <[email protected]> wrote:
> >
> > Thanks for pointing it out. Here is the new version patch which add this version check.
>
> I have added/edited a few comments and ran pgindent. The attached
> looks good to me. I'll push this early next week unless there are more
> comments/suggestions.
A minor doc review.
Note that I also sent some doc comments at [email protected].
+ lists among publications in which case <command>ALTER PUBLICATION</command>
+ command will be successful but later the WalSender in publisher or the
COMMA in which
remove "command" ?
s/in publisher/on the publisher/
+ Subscription having several publications in which the same table has been
+ published with different column lists is not supported.
Either "Subscriptions having .. are not supported"; or,
"A subscription having .. is not supported".
^ permalink raw reply [nested|flat] 58+ messages in thread
* RE: bogus: logical replication rows/cols combinations
@ 2022-05-27 07:34 [email protected] <[email protected]>
parent: Justin Pryzby <[email protected]>
0 siblings, 1 reply; 58+ messages in thread
From: [email protected] @ 2022-05-27 07:34 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; Amit Kapila <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Tomas Vondra <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Tom Lane <[email protected]>
On Friday, May 27, 2022 1:54 PM Justin Pryzby <[email protected]> wrote:
>
> On Fri, May 27, 2022 at 11:17:00AM +0530, Amit Kapila wrote:
> > On Tue, May 24, 2022 at 11:03 AM [email protected]
> <[email protected]> wrote:
> > >
> > > On Friday, May 20, 2022 11:06 AM Amit Kapila <[email protected]>
> wrote:
> > >
> > > Thanks for pointing it out. Here is the new version patch which add this
> version check.
> >
> > I have added/edited a few comments and ran pgindent. The attached
> > looks good to me. I'll push this early next week unless there are more
> > comments/suggestions.
>
> A minor doc review.
> Note that I also sent some doc comments at
> [email protected].
>
> + lists among publications in which case <command>ALTER
> PUBLICATION</command>
> + command will be successful but later the WalSender in publisher
> + or the
>
> COMMA in which
>
> remove "command" ?
>
> s/in publisher/on the publisher/
>
> + Subscription having several publications in which the same table has been
> + published with different column lists is not supported.
>
> Either "Subscriptions having .. are not supported"; or, "A subscription having ..
> is not supported".
Thanks for the comments. Here is the new version patch set which fixes these.
Best regards,
Hou zj
Attachments:
[application/octet-stream] 0001-language-fixes-on-HEAD-from-Justin.patch (3.0K, ../../OS0PR01MB5716FFBE5AACA8707C3420E994D89@OS0PR01MB5716.jpnprd01.prod.outlook.com/2-0001-language-fixes-on-HEAD-from-Justin.patch)
download | inline diff:
From 83a6e85100a173124396f8d4ac29ff170319f0a4 Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <[email protected]>
Date: Fri, 27 May 2022 15:24:52 +0800
Subject: [PATCH] language fixes from Justin
---
doc/src/sgml/catalogs.sgml | 6 +++---
src/backend/catalog/pg_publication.c | 2 +-
src/include/catalog/pg_proc.dat | 2 +-
3 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index c00c93d..9808180 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -9701,7 +9701,7 @@ SCRAM-SHA-256$<replaceable><iteration count></replaceable>:<replaceable>&l
<row>
<entry><link linkend="view-pg-publication-tables"><structname>pg_publication_tables</structname></link></entry>
- <entry>publications and information of their associated tables</entry>
+ <entry>publications and information about their associated tables</entry>
</row>
<row>
@@ -11645,7 +11645,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
<para>
The view <structname>pg_publication_tables</structname> provides
- information about the mapping between publications and information of
+ information about the mapping between publications and information about
tables they contain. Unlike the underlying catalog
<link linkend="catalog-pg-publication-rel"><structname>pg_publication_rel</structname></link>,
this view expands publications defined as <literal>FOR ALL TABLES</literal>
@@ -11705,7 +11705,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
</para>
<para>
Names of table columns included in the publication. This contains all
- the columns of the table when the user didn't specify the column list
+ the columns of the table when the user didn't specify a column list
for the table.
</para></entry>
</row>
diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c
index 8c7fca6..2f706f6 100644
--- a/src/backend/catalog/pg_publication.c
+++ b/src/backend/catalog/pg_publication.c
@@ -1077,7 +1077,7 @@ get_publication_name(Oid pubid, bool missing_ok)
}
/*
- * Returns information of tables in a publication.
+ * Returns information about tables in a publication.
*/
Datum
pg_get_publication_tables(PG_FUNCTION_ARGS)
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 87aa571..86f1329 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -11673,7 +11673,7 @@
prosrc => 'pg_show_replication_origin_status' },
# publications
-{ oid => '6119', descr => 'get information of tables in a publication',
+{ oid => '6119', descr => 'get information about tables in a publication',
proname => 'pg_get_publication_tables', prorows => '1000', proretset => 't',
provolatile => 's', prorettype => 'record', proargtypes => 'text',
proallargtypes => '{text,oid,int2vector,pg_node_tree}', proargmodes => '{i,o,o,o}',
--
2.7.2.windows.1
[application/octet-stream] v6-0001-Prohibit-combining-publications-with-different-co.patch (22.4K, ../../OS0PR01MB5716FFBE5AACA8707C3420E994D89@OS0PR01MB5716.jpnprd01.prod.outlook.com/3-v6-0001-Prohibit-combining-publications-with-different-co.patch)
download | inline diff:
From 201721776782809c3bb99bb6ff8fbdedb02483ab Mon Sep 17 00:00:00 2001
From: Amit Kapila <[email protected]>
Date: Fri, 27 May 2022 08:30:33 +0530
Subject: [PATCH v5] Prohibit combining publications with different column
lists.
The main purpose of introducing a column list is to have statically
different shapes on publisher and subscriber or hide sensitive column
data. In both cases, it doesn't seem to make sense to combine column
lists.
So, we disallow the cases where the column list is different for the same
table when combining publications. It can be later extended to combine the
column lists for selective cases where required.
Reported-by: Alvaro Herrera
Author: Hou Zhijie
Reviewed-by: Amit Kapila, Alvaro Herrera, Shi Yu, Takamichi Osumi, Justin Pryzby
Discussion: https://postgr.es/m/[email protected]
---
doc/src/sgml/ref/alter_publication.sgml | 12 ++-
doc/src/sgml/ref/create_subscription.sgml | 5 +
src/backend/commands/subscriptioncmds.c | 28 +++++-
src/backend/replication/logical/tablesync.c | 72 ++++++-------
src/backend/replication/pgoutput/pgoutput.c | 80 +++++++--------
src/test/subscription/t/031_column_list.pl | 150 ++++++++++++----------------
6 files changed, 181 insertions(+), 166 deletions(-)
diff --git a/doc/src/sgml/ref/alter_publication.sgml b/doc/src/sgml/ref/alter_publication.sgml
index e2cce49..f03933a 100644
--- a/doc/src/sgml/ref/alter_publication.sgml
+++ b/doc/src/sgml/ref/alter_publication.sgml
@@ -116,7 +116,17 @@ ALTER PUBLICATION <replaceable class="parameter">name</replaceable> RENAME TO <r
<para>
Optionally, a column list can be specified. See <xref
- linkend="sql-createpublication"/> for details.
+ linkend="sql-createpublication"/> for details. Note that a subscription
+ having several publications in which the same table has been published
+ with different column lists is not supported. So, changing the column
+ lists of the tables being subscribed could cause inconsistency of column
+ lists among publications, in which case <command>ALTER PUBLICATION</command>
+ will be successful but later the WalSender on the publisher or the
+ subscriber may throw an error. In this scenario, the user needs to
+ recreate the subscription after adjusting the column list or drop the
+ problematic publication using
+ <literal>ALTER SUBSCRIPTION ... DROP PUBLICATION</literal> and then add
+ it back after adjusting the column list.
</para>
<para>
diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml
index 203bb41..f6f82a0 100644
--- a/doc/src/sgml/ref/create_subscription.sgml
+++ b/doc/src/sgml/ref/create_subscription.sgml
@@ -356,6 +356,11 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl
</para>
<para>
+ Subscriptions having several publications in which the same table has been
+ published with different column lists are not supported.
+ </para>
+
+ <para>
We allow non-existent publications to be specified so that users can add
those later. This means
<link linkend="catalog-pg-subscription"><structname>pg_subscription</structname></link>
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 690cdaa..83e6eae 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -1754,6 +1754,11 @@ AlterSubscriptionOwner_oid(Oid subid, Oid newOwnerId)
/*
* Get the list of tables which belong to specified publications on the
* publisher connection.
+ *
+ * Note that we don't support the case where the column list is different for
+ * the same table in different publications to avoid sending unwanted column
+ * information for some of the rows. This can happen when both the column
+ * list and row filter are specified for different publications.
*/
static List *
fetch_table_list(WalReceiverConn *wrconn, List *publications)
@@ -1761,17 +1766,23 @@ fetch_table_list(WalReceiverConn *wrconn, List *publications)
WalRcvExecResult *res;
StringInfoData cmd;
TupleTableSlot *slot;
- Oid tableRow[2] = {TEXTOID, TEXTOID};
+ Oid tableRow[3] = {TEXTOID, TEXTOID, NAMEARRAYOID};
List *tablelist = NIL;
+ bool check_columnlist = (walrcv_server_version(wrconn) >= 150000);
initStringInfo(&cmd);
- appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename\n"
- " FROM pg_catalog.pg_publication_tables t\n"
+ appendStringInfoString(&cmd, "SELECT DISTINCT t.schemaname, t.tablename \n");
+
+ /* Get column lists for each relation if the publisher supports it */
+ if (check_columnlist)
+ appendStringInfoString(&cmd, ", t.attnames\n");
+
+ appendStringInfoString(&cmd, "FROM pg_catalog.pg_publication_tables t\n"
" WHERE t.pubname IN (");
get_publications_str(publications, &cmd, true);
appendStringInfoChar(&cmd, ')');
- res = walrcv_exec(wrconn, cmd.data, 2, tableRow);
+ res = walrcv_exec(wrconn, cmd.data, check_columnlist ? 3 : 2, tableRow);
pfree(cmd.data);
if (res->status != WALRCV_OK_TUPLES)
@@ -1795,7 +1806,14 @@ fetch_table_list(WalReceiverConn *wrconn, List *publications)
Assert(!isnull);
rv = makeRangeVar(nspname, relname, -1);
- tablelist = lappend(tablelist, rv);
+
+ if (check_columnlist && list_member(tablelist, rv))
+ ereport(ERROR,
+ errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot use different column lists for table \"%s.%s\" in different publications",
+ nspname, relname));
+ else
+ tablelist = lappend(tablelist, rv);
ExecClearTuple(slot);
}
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 994c7a0..670c6fc 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -753,17 +753,6 @@ fetch_remote_table_info(char *nspname, char *relname,
/*
* Get column lists for each relation.
*
- * For initial synchronization, column lists can be ignored in following
- * cases:
- *
- * 1) one of the subscribed publications for the table hasn't specified
- * any column list
- *
- * 2) one of the subscribed publications has puballtables set to true
- *
- * 3) one of the subscribed publications is declared as ALL TABLES IN
- * SCHEMA that includes this relation
- *
* We need to do this before fetching info about column names and types,
* so that we can skip columns that should not be replicated.
*/
@@ -771,7 +760,7 @@ fetch_remote_table_info(char *nspname, char *relname,
{
WalRcvExecResult *pubres;
TupleTableSlot *slot;
- Oid attrsRow[] = {INT2OID};
+ Oid attrsRow[] = {INT2VECTOROID};
StringInfoData pub_names;
bool first = true;
@@ -786,19 +775,17 @@ fetch_remote_table_info(char *nspname, char *relname,
/*
* Fetch info about column lists for the relation (from all the
- * publications). We unnest the int2vector values, because that makes
- * it easier to combine lists by simply adding the attnums to a new
- * bitmap (without having to parse the int2vector data). This
- * preserves NULL values, so that if one of the publications has no
- * column list, we'll know that.
+ * publications).
*/
resetStringInfo(&cmd);
appendStringInfo(&cmd,
- "SELECT DISTINCT unnest"
+ "SELECT DISTINCT"
+ " (CASE WHEN (array_length(gpt.attrs, 1) = c.relnatts)"
+ " THEN NULL ELSE gpt.attrs END)"
" FROM pg_publication p,"
- " LATERAL pg_get_publication_tables(p.pubname) gpt"
- " LEFT OUTER JOIN unnest(gpt.attrs) ON TRUE"
- " WHERE gpt.relid = %u"
+ " LATERAL pg_get_publication_tables(p.pubname) gpt,"
+ " pg_class c"
+ " WHERE gpt.relid = %u AND c.oid = gpt.relid"
" AND p.pubname IN ( %s )",
lrel->remoteid,
pub_names.data);
@@ -813,26 +800,43 @@ fetch_remote_table_info(char *nspname, char *relname,
nspname, relname, pubres->err)));
/*
- * Merge the column lists (from different publications) by creating a
- * single bitmap with all the attnums. If we find a NULL value, that
- * means one of the publications has no column list for the table
- * we're syncing.
+ * We don't support the case where the column list is different for
+ * the same table when combining publications. See comments atop
+ * fetch_table_list. So there should be only one row returned.
+ * Although we already checked this when creating the subscription, we
+ * still need to check here in case the column list was changed after
+ * creating the subscription and before the sync worker is started.
+ */
+ if (tuplestore_tuple_count(pubres->tuplestore) > 1)
+ ereport(ERROR,
+ errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot use different column lists for table \"%s.%s\" in different publications",
+ nspname, relname));
+
+ /*
+ * Get the column list and build a single bitmap with the attnums.
+ *
+ * If we find a NULL value, it means all the columns should be
+ * replicated.
*/
slot = MakeSingleTupleTableSlot(pubres->tupledesc, &TTSOpsMinimalTuple);
- while (tuplestore_gettupleslot(pubres->tuplestore, true, false, slot))
+ if (tuplestore_gettupleslot(pubres->tuplestore, true, false, slot))
{
Datum cfval = slot_getattr(slot, 1, &isnull);
- /* NULL means empty column list, so we're done. */
- if (isnull)
+ if (!isnull)
{
- bms_free(included_cols);
- included_cols = NULL;
- break;
- }
+ ArrayType *arr;
+ int nelems;
+ int16 *elems;
- included_cols = bms_add_member(included_cols,
- DatumGetInt16(cfval));
+ arr = DatumGetArrayTypeP(cfval);
+ nelems = ARR_DIMS(arr)[0];
+ elems = (int16 *) ARR_DATA_PTR(arr);
+
+ for (natt = 0; natt < nelems; natt++)
+ included_cols = bms_add_member(included_cols, elems[natt]);
+ }
ExecClearTuple(slot);
}
diff --git a/src/backend/replication/pgoutput/pgoutput.c b/src/backend/replication/pgoutput/pgoutput.c
index 42c06af..8deae57 100644
--- a/src/backend/replication/pgoutput/pgoutput.c
+++ b/src/backend/replication/pgoutput/pgoutput.c
@@ -979,30 +979,31 @@ pgoutput_column_list_init(PGOutputData *data, List *publications,
RelationSyncEntry *entry)
{
ListCell *lc;
+ bool first = true;
+ Relation relation = RelationIdGetRelation(entry->publish_as_relid);
/*
* Find if there are any column lists for this relation. If there are,
- * build a bitmap merging all the column lists.
- *
- * All the given publication-table mappings must be checked.
+ * build a bitmap using the column lists.
*
* Multiple publications might have multiple column lists for this
* relation.
*
+ * Note that we don't support the case where the column list is different
+ * for the same table when combining publications. See comments atop
+ * fetch_table_list. But one can later change the publication so we still
+ * need to check all the given publication-table mappings and report an
+ * error if any publications have a different column list.
+ *
* FOR ALL TABLES and FOR ALL TABLES IN SCHEMA implies "don't use column
- * list" so it takes precedence.
+ * list".
*/
foreach(lc, publications)
{
Publication *pub = lfirst(lc);
HeapTuple cftuple = NULL;
Datum cfdatum = 0;
-
- /*
- * Assume there's no column list. Only if we find pg_publication_rel
- * entry with a column list we'll switch it to false.
- */
- bool pub_no_list = true;
+ Bitmapset *cols = NULL;
/*
* If the publication is FOR ALL TABLES then it is treated the same as
@@ -1011,6 +1012,8 @@ pgoutput_column_list_init(PGOutputData *data, List *publications,
*/
if (!pub->alltables)
{
+ bool pub_no_list = true;
+
/*
* Check for the presence of a column list in this publication.
*
@@ -1024,51 +1027,48 @@ pgoutput_column_list_init(PGOutputData *data, List *publications,
if (HeapTupleIsValid(cftuple))
{
- /*
- * Lookup the column list attribute.
- *
- * Note: We update the pub_no_list value directly, because if
- * the value is NULL, we have no list (and vice versa).
- */
+ /* Lookup the column list attribute. */
cfdatum = SysCacheGetAttr(PUBLICATIONRELMAP, cftuple,
Anum_pg_publication_rel_prattrs,
&pub_no_list);
- /*
- * Build the column list bitmap in the per-entry context.
- *
- * We need to merge column lists from all publications, so we
- * update the same bitmapset. If the column list is null, we
- * interpret it as replicating all columns.
- */
+ /* Build the column list bitmap in the per-entry context. */
if (!pub_no_list) /* when not null */
{
pgoutput_ensure_entry_cxt(data, entry);
- entry->columns = pub_collist_to_bitmapset(entry->columns,
- cfdatum,
- entry->entry_cxt);
+ cols = pub_collist_to_bitmapset(cols, cfdatum,
+ entry->entry_cxt);
+
+ /*
+ * If column list includes all the columns of the table,
+ * set it to NULL.
+ */
+ if (bms_num_members(cols) == RelationGetNumberOfAttributes(relation))
+ {
+ bms_free(cols);
+ cols = NULL;
+ }
}
+
+ ReleaseSysCache(cftuple);
}
}
- /*
- * Found a publication with no column list, so we're done. But first
- * discard column list we might have from preceding publications.
- */
- if (pub_no_list)
+ if (first)
{
- if (cftuple)
- ReleaseSysCache(cftuple);
-
- bms_free(entry->columns);
- entry->columns = NULL;
-
- break;
+ entry->columns = cols;
+ first = false;
}
-
- ReleaseSysCache(cftuple);
+ else if (!bms_equal(entry->columns, cols))
+ ereport(ERROR,
+ errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot use different column lists for table \"%s.%s\" in different publications",
+ get_namespace_name(RelationGetNamespace(relation)),
+ RelationGetRelationName(relation)));
} /* loop all subscribed publications */
+
+ RelationClose(relation);
}
/*
diff --git a/src/test/subscription/t/031_column_list.pl b/src/test/subscription/t/031_column_list.pl
index 19812e1..7f031bc 100644
--- a/src/test/subscription/t/031_column_list.pl
+++ b/src/test/subscription/t/031_column_list.pl
@@ -20,6 +20,7 @@ $node_subscriber->append_conf('postgresql.conf',
$node_subscriber->start;
my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres';
+my $offset = 0;
sub wait_for_subscription_sync
{
@@ -361,13 +362,13 @@ is( $result, qq(1|abc
2|xyz), 'update on column tab2.c is not replicated');
-# TEST: add a table to two publications with different column lists, and
+# TEST: add a table to two publications with same column lists, and
# create a single subscription replicating both publications
$node_publisher->safe_psql(
'postgres', qq(
CREATE TABLE tab5 (a int PRIMARY KEY, b int, c int, d int);
CREATE PUBLICATION pub2 FOR TABLE tab5 (a, b);
- CREATE PUBLICATION pub3 FOR TABLE tab5 (a, d);
+ CREATE PUBLICATION pub3 FOR TABLE tab5 (a, b);
-- insert a couple initial records
INSERT INTO tab5 VALUES (1, 11, 111, 1111);
@@ -388,8 +389,7 @@ wait_for_subscription_sync($node_subscriber);
$node_publisher->wait_for_catchup('sub1');
-# insert data and make sure all the columns (union of the columns lists)
-# get fully replicated
+# insert data and make sure the columns in column list get fully replicated
$node_publisher->safe_psql(
'postgres', qq(
INSERT INTO tab5 VALUES (3, 33, 333, 3333);
@@ -399,42 +399,12 @@ $node_publisher->safe_psql(
$node_publisher->wait_for_catchup('sub1');
is( $node_subscriber->safe_psql('postgres', "SELECT * FROM tab5 ORDER BY a"),
- qq(1|11|1111
-2|22|2222
-3|33|3333
-4|44|4444),
+ qq(1|11|
+2|22|
+3|33|
+4|44|),
'overlapping publications with overlapping column lists');
-# and finally, remove the column list for one of the publications, which
-# means replicating all columns (removing the column list), but first add
-# the missing column to the table on subscriber
-$node_publisher->safe_psql(
- 'postgres', qq(
- ALTER PUBLICATION pub3 SET TABLE tab5;
-));
-
-$node_subscriber->safe_psql(
- 'postgres', qq(
- ALTER SUBSCRIPTION sub1 REFRESH PUBLICATION;
- ALTER TABLE tab5 ADD COLUMN c INT;
-));
-
-wait_for_subscription_sync($node_subscriber);
-
-$node_publisher->safe_psql(
- 'postgres', qq(
- INSERT INTO tab5 VALUES (5, 55, 555, 5555);
-));
-
-$node_publisher->wait_for_catchup('sub1');
-
-is( $node_subscriber->safe_psql('postgres', "SELECT * FROM tab5 ORDER BY a"),
- qq(1|11|1111|
-2|22|2222|
-3|33|3333|
-4|44|4444|
-5|55|5555|555),
- 'overlapping publications with overlapping column lists');
# TEST: create a table with a column list, then change the replica
# identity by replacing a primary key (but use a different column in
@@ -900,57 +870,21 @@ is( $node_subscriber->safe_psql(
3|),
'partitions with different replica identities not replicated correctly');
-# TEST: With a table included in multiple publications, we should use a
-# union of the column lists. So with column lists (a,b) and (a,c) we
-# should replicate (a,b,c).
-
-$node_publisher->safe_psql(
- 'postgres', qq(
- CREATE TABLE test_mix_1 (a int PRIMARY KEY, b int, c int);
- CREATE PUBLICATION pub_mix_1 FOR TABLE test_mix_1 (a, b);
- CREATE PUBLICATION pub_mix_2 FOR TABLE test_mix_1 (a, c);
-
- -- initial data
- INSERT INTO test_mix_1 VALUES (1, 2, 3);
-));
-
-$node_subscriber->safe_psql(
- 'postgres', qq(
- CREATE TABLE test_mix_1 (a int PRIMARY KEY, b int, c int);
- ALTER SUBSCRIPTION sub1 SET PUBLICATION pub_mix_1, pub_mix_2;
-));
-
-wait_for_subscription_sync($node_subscriber);
-
-$node_publisher->safe_psql(
- 'postgres', qq(
- INSERT INTO test_mix_1 VALUES (4, 5, 6);
-));
-
-$node_publisher->wait_for_catchup('sub1');
-
-is( $node_subscriber->safe_psql(
- 'postgres', "SELECT * FROM test_mix_1 ORDER BY a"),
- qq(1|2|3
-4|5|6),
- 'a mix of publications should use a union of column list');
-
-# TEST: With a table included in multiple publications, we should use a
-# union of the column lists. If any of the publications is FOR ALL
-# TABLES, we should replicate all columns.
+# TEST: With a table included in the publications which is FOR ALL TABLES, it
+# means replicate all columns.
# drop unnecessary tables, so as not to interfere with the FOR ALL TABLES
$node_publisher->safe_psql(
'postgres', qq(
- DROP TABLE tab1, tab2, tab3, tab4, tab5, tab6, tab7, test_mix_1,
+ DROP TABLE tab1, tab2, tab3, tab4, tab5, tab6, tab7,
test_part, test_part_a, test_part_b, test_part_c, test_part_d;
));
$node_publisher->safe_psql(
'postgres', qq(
CREATE TABLE test_mix_2 (a int PRIMARY KEY, b int, c int);
- CREATE PUBLICATION pub_mix_3 FOR TABLE test_mix_2 (a, b);
+ CREATE PUBLICATION pub_mix_3 FOR TABLE test_mix_2 (a, b, c);
CREATE PUBLICATION pub_mix_4 FOR ALL TABLES;
-- initial data
@@ -976,12 +910,11 @@ $node_publisher->wait_for_catchup('sub1');
is( $node_subscriber->safe_psql('postgres', "SELECT * FROM test_mix_2"),
qq(1|2|3
4|5|6),
- 'a mix of publications should use a union of column list');
+ 'all columns should be replicated');
-# TEST: With a table included in multiple publications, we should use a
-# union of the column lists. If any of the publications is FOR ALL
-# TABLES IN SCHEMA, we should replicate all columns.
+# TEST: With a table included in the publication which is FOR ALL TABLES
+# IN SCHEMA, it means replicate all columns.
$node_subscriber->safe_psql(
'postgres', qq(
@@ -993,7 +926,7 @@ $node_publisher->safe_psql(
'postgres', qq(
DROP TABLE test_mix_2;
CREATE TABLE test_mix_3 (a int PRIMARY KEY, b int, c int);
- CREATE PUBLICATION pub_mix_5 FOR TABLE test_mix_3 (a, b);
+ CREATE PUBLICATION pub_mix_5 FOR TABLE test_mix_3 (a, b, c);
CREATE PUBLICATION pub_mix_6 FOR ALL TABLES IN SCHEMA public;
-- initial data
@@ -1017,7 +950,7 @@ $node_publisher->wait_for_catchup('sub1');
is( $node_subscriber->safe_psql('postgres', "SELECT * FROM test_mix_3"),
qq(1|2|3
4|5|6),
- 'a mix of publications should use a union of column list');
+ 'all columns should be replicated');
# TEST: Check handling of publish_via_partition_root - if a partition is
@@ -1074,7 +1007,7 @@ is( $node_subscriber->safe_psql(
# TEST: Multiple publications which publish schema of parent table and
# partition. The partition is published through two publications, once
# through a schema (so no column list) containing the parent, and then
-# also directly (with a columns list). The expected outcome is there is
+# also directly (with all columns). The expected outcome is there is
# no column list.
$node_publisher->safe_psql(
@@ -1086,7 +1019,7 @@ $node_publisher->safe_psql(
CREATE TABLE t_1 PARTITION OF s1.t FOR VALUES FROM (1) TO (10);
CREATE PUBLICATION pub1 FOR ALL TABLES IN SCHEMA s1;
- CREATE PUBLICATION pub2 FOR TABLE t_1(b);
+ CREATE PUBLICATION pub2 FOR TABLE t_1(a, b, c);
-- initial data
INSERT INTO s1.t VALUES (1, 2, 3);
@@ -1233,6 +1166,51 @@ is( $node_subscriber->safe_psql(
'publication containing both parent and child relation');
+# TEST: With a table included in multiple publications with different column
+# lists, we should catch the error when creating the subscription.
+
+$node_publisher->safe_psql('postgres', qq(
+ CREATE TABLE test_mix_1 (a int PRIMARY KEY, b int, c int);
+ CREATE PUBLICATION pub_mix_1 FOR TABLE test_mix_1 (a, b);
+ CREATE PUBLICATION pub_mix_2 FOR TABLE test_mix_1 (a, c);
+));
+
+$node_subscriber->safe_psql('postgres', qq(
+ DROP SUBSCRIPTION sub1;
+ CREATE TABLE test_mix_1 (a int PRIMARY KEY, b int, c int);
+));
+
+my ($cmdret, $stdout, $stderr) = $node_subscriber->psql(
+ 'postgres', qq(
+ CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_mix_1, pub_mix_2;
+));
+
+ok( $stderr =~
+ qr/cannot use different column lists for table "public.test_mix_1" in different publications/,
+ 'different column lists detected');
+
+# TEST: If the column list is changed after creating the subscription, we
+# should catch the error reported by walsender.
+
+$node_publisher->safe_psql('postgres', qq(
+ ALTER PUBLICATION pub_mix_1 SET TABLE test_mix_1 (a, c);
+));
+
+$node_subscriber->safe_psql('postgres', qq(
+ CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub_mix_1, pub_mix_2;
+));
+
+$node_publisher->wait_for_catchup('sub1');
+
+$node_publisher->safe_psql('postgres', qq(
+ ALTER PUBLICATION pub_mix_1 SET TABLE test_mix_1 (a, b);
+ INSERT INTO test_mix_1 VALUES(1, 1, 1);
+));
+
+$offset = $node_publisher->wait_for_log(
+ qr/cannot use different column lists for table "public.test_mix_1" in different publications/,
+ $offset);
+
$node_subscriber->stop('fast');
$node_publisher->stop('fast');
--
1.8.3.1
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-06-02 11:58 Amit Kapila <[email protected]>
parent: [email protected] <[email protected]>
0 siblings, 1 reply; 58+ messages in thread
From: Amit Kapila @ 2022-06-02 11:58 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Justin Pryzby <[email protected]>; Alvaro Herrera <[email protected]>; Tomas Vondra <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Tom Lane <[email protected]>
On Fri, May 27, 2022 at 1:04 PM [email protected]
<[email protected]> wrote:
>
> On Friday, May 27, 2022 1:54 PM Justin Pryzby <[email protected]> wrote:
> >
> > On Fri, May 27, 2022 at 11:17:00AM +0530, Amit Kapila wrote:
> > > On Tue, May 24, 2022 at 11:03 AM [email protected]
> > <[email protected]> wrote:
> > > >
> > > > On Friday, May 20, 2022 11:06 AM Amit Kapila <[email protected]>
> > wrote:
> > > >
> > > > Thanks for pointing it out. Here is the new version patch which add this
> > version check.
> > >
> > > I have added/edited a few comments and ran pgindent. The attached
> > > looks good to me. I'll push this early next week unless there are more
> > > comments/suggestions.
> >
> > A minor doc review.
> > Note that I also sent some doc comments at
> > [email protected].
> >
> > + lists among publications in which case <command>ALTER
> > PUBLICATION</command>
> > + command will be successful but later the WalSender in publisher
> > + or the
> >
> > COMMA in which
> >
> > remove "command" ?
> >
> > s/in publisher/on the publisher/
> >
> > + Subscription having several publications in which the same table has been
> > + published with different column lists is not supported.
> >
> > Either "Subscriptions having .. are not supported"; or, "A subscription having ..
> > is not supported".
>
> Thanks for the comments. Here is the new version patch set which fixes these.
>
I have pushed the bug-fix patch. I'll look at the language
improvements patch next.
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-06-06 05:42 Peter Smith <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 1 reply; 58+ messages in thread
From: Peter Smith @ 2022-06-06 05:42 UTC (permalink / raw)
To: Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; Justin Pryzby <[email protected]>; Alvaro Herrera <[email protected]>; Tomas Vondra <[email protected]>; Peter Eisentraut <[email protected]>; [email protected] <[email protected]>; Tom Lane <[email protected]>
On Thu, Jun 2, 2022 at 9:58 PM Amit Kapila <[email protected]> wrote:
>
> On Fri, May 27, 2022 at 1:04 PM [email protected]
> <[email protected]> wrote:
> >
> > On Friday, May 27, 2022 1:54 PM Justin Pryzby <[email protected]> wrote:
> > >
> > > On Fri, May 27, 2022 at 11:17:00AM +0530, Amit Kapila wrote:
> > > > On Tue, May 24, 2022 at 11:03 AM [email protected]
> > > <[email protected]> wrote:
> > > > >
> > > > > On Friday, May 20, 2022 11:06 AM Amit Kapila <[email protected]>
> > > wrote:
> > > > >
> > > > > Thanks for pointing it out. Here is the new version patch which add this
> > > version check.
> > > >
> > > > I have added/edited a few comments and ran pgindent. The attached
> > > > looks good to me. I'll push this early next week unless there are more
> > > > comments/suggestions.
> > >
> > > A minor doc review.
> > > Note that I also sent some doc comments at
> > > [email protected].
> > >
> > > + lists among publications in which case <command>ALTER
> > > PUBLICATION</command>
> > > + command will be successful but later the WalSender in publisher
> > > + or the
> > >
> > > COMMA in which
> > >
> > > remove "command" ?
> > >
> > > s/in publisher/on the publisher/
> > >
> > > + Subscription having several publications in which the same table has been
> > > + published with different column lists is not supported.
> > >
> > > Either "Subscriptions having .. are not supported"; or, "A subscription having ..
> > > is not supported".
> >
> > Thanks for the comments. Here is the new version patch set which fixes these.
> >
>
> I have pushed the bug-fix patch. I'll look at the language
> improvements patch next.
I noticed the patch "0001-language-fixes-on-HEAD-from-Justin.patch" says:
@@ -11673,7 +11673,7 @@
prosrc => 'pg_show_replication_origin_status' },
# publications
-{ oid => '6119', descr => 'get information of tables in a publication',
+{ oid => '6119', descr => 'get information about tables in a publication',
~~~
But, this grammar website [1] says:
What Does Of Mean
As defined by Cambridge dictionary Of is basically used “to show
possession, belonging, or origin”.
What Does About Mean
Similarly about primarily indicates ‘On the subject of; concerning’ as
defined by the Oxford dictionary. Or about in brief highlights some
fact ‘on the subject of, or connected with’
The main difference between of and about is that of implies a
possessive quality while about implies concerning or on the subject of
something.
~~~
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-06-08 03:25 Justin Pryzby <[email protected]>
parent: Peter Smith <[email protected]>
0 siblings, 1 reply; 58+ messages in thread
From: Justin Pryzby @ 2022-06-08 03:25 UTC (permalink / raw)
To: Peter Smith <[email protected]>; +Cc: Amit Kapila <[email protected]>; [email protected]; Alvaro Herrera <[email protected]>; Tomas Vondra <[email protected]>; Peter Eisentraut <[email protected]>; [email protected]; Tom Lane <[email protected]>
On Mon, Jun 06, 2022 at 03:42:31PM +1000, Peter Smith wrote:
> I noticed the patch "0001-language-fixes-on-HEAD-from-Justin.patch" says:
>
> @@ -11673,7 +11673,7 @@
> prosrc => 'pg_show_replication_origin_status' },
>
> # publications
> -{ oid => '6119', descr => 'get information of tables in a publication',
> +{ oid => '6119', descr => 'get information about tables in a publication',
>
> ~~~
>
> But, this grammar website [1] says:
...
> From which I guess
>
> 1. 'get information of tables in a publication' ~= 'get information
> belonging to tables in a publication'
But the information doesn't "belong to" the tables.
The information is "regarding" the tables (or "associated with" or "concerned
with" or "respecting" or "on the subject of" the tables).
I think my change is correct based on the grammar definition, as well as its
intuitive "feel".
--
Justin
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-06-08 05:35 Peter Smith <[email protected]>
parent: Justin Pryzby <[email protected]>
0 siblings, 1 reply; 58+ messages in thread
From: Peter Smith @ 2022-06-08 05:35 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: Amit Kapila <[email protected]>; [email protected]; Alvaro Herrera <[email protected]>; Tomas Vondra <[email protected]>; Peter Eisentraut <[email protected]>; [email protected]; Tom Lane <[email protected]>
On Wed, Jun 8, 2022 at 1:25 PM Justin Pryzby <[email protected]> wrote:
>
> On Mon, Jun 06, 2022 at 03:42:31PM +1000, Peter Smith wrote:
> > I noticed the patch "0001-language-fixes-on-HEAD-from-Justin.patch" says:
> >
> > @@ -11673,7 +11673,7 @@
> > prosrc => 'pg_show_replication_origin_status' },
> >
> > # publications
> > -{ oid => '6119', descr => 'get information of tables in a publication',
> > +{ oid => '6119', descr => 'get information about tables in a publication',
> >
> > ~~~
> >
> > But, this grammar website [1] says:
> ...
> > From which I guess
> >
> > 1. 'get information of tables in a publication' ~= 'get information
> > belonging to tables in a publication'
>
> But the information doesn't "belong to" the tables.
>
> The information is "regarding" the tables (or "associated with" or "concerned
> with" or "respecting" or "on the subject of" the tables).
>
> I think my change is correct based on the grammar definition, as well as its
> intuitive "feel".
>
Actually, I have no problem with this being worded either way. My
point was mostly to question if it was really worth changing it at
this time - e.g. I think there is a reluctance to change anything to
do with the catalogs during beta (even when a catversion bump may not
be required).
I agree that "about" seems better if the text said, "get information
about tables". But it does not say that - it says "get information
about tables in a publication" which I felt made a subtle difference.
e.g.1 "... on the subject of / concerned with tables."
- sounds like attributes about each table (col names, row filter etc)
versus
e.g.2 "... on the subject of / concerned with tables in a publication."
- sounds less like information PER table, and more like information
about the table membership of the publication.
~~
Any ambiguities can be eliminated if this text was just fixed to be
consistent with the wording of catalogs.sgml:
e.g. "publications and information about their associated tables"
But then this comes full circle back to my question if during beta is
a good time to be making such a change.
------
Kind Regards,
Peter Smith.
Fujitsu Australia
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-06-13 03:24 Amit Kapila <[email protected]>
parent: Peter Smith <[email protected]>
0 siblings, 1 reply; 58+ messages in thread
From: Amit Kapila @ 2022-06-13 03:24 UTC (permalink / raw)
To: Peter Smith <[email protected]>; +Cc: Justin Pryzby <[email protected]>; Hou, Zhijie/侯 志杰 <[email protected]>; Alvaro Herrera <[email protected]>; Tomas Vondra <[email protected]>; Peter Eisentraut <[email protected]>; PostgreSQL Hackers <[email protected]>; Tom Lane <[email protected]>
On Wed, Jun 8, 2022 at 11:05 AM Peter Smith <[email protected]> wrote:
>
> On Wed, Jun 8, 2022 at 1:25 PM Justin Pryzby <[email protected]> wrote:
> >
> > On Mon, Jun 06, 2022 at 03:42:31PM +1000, Peter Smith wrote:
> > > I noticed the patch "0001-language-fixes-on-HEAD-from-Justin.patch" says:
> > >
> > > @@ -11673,7 +11673,7 @@
> > > prosrc => 'pg_show_replication_origin_status' },
> > >
> > > # publications
> > > -{ oid => '6119', descr => 'get information of tables in a publication',
> > > +{ oid => '6119', descr => 'get information about tables in a publication',
> > >
> > > ~~~
> > >
> > > But, this grammar website [1] says:
> > ...
> > > From which I guess
> > >
> > > 1. 'get information of tables in a publication' ~= 'get information
> > > belonging to tables in a publication'
> >
> > But the information doesn't "belong to" the tables.
> >
> > The information is "regarding" the tables (or "associated with" or "concerned
> > with" or "respecting" or "on the subject of" the tables).
> >
> > I think my change is correct based on the grammar definition, as well as its
> > intuitive "feel".
> >
>
> Actually, I have no problem with this being worded either way. My
> point was mostly to question if it was really worth changing it at
> this time - e.g. I think there is a reluctance to change anything to
> do with the catalogs during beta (even when a catversion bump may not
> be required).
>
> I agree that "about" seems better if the text said, "get information
> about tables". But it does not say that - it says "get information
> about tables in a publication" which I felt made a subtle difference.
>
> e.g.1 "... on the subject of / concerned with tables."
> - sounds like attributes about each table (col names, row filter etc)
>
> versus
>
> e.g.2 "... on the subject of / concerned with tables in a publication."
> - sounds less like information PER table, and more like information
> about the table membership of the publication.
>
> ~~
>
> Any ambiguities can be eliminated if this text was just fixed to be
> consistent with the wording of catalogs.sgml:
> e.g. "publications and information about their associated tables"
>
I don't know if this is better than the current text for this view:
'get information of tables in a publication' and unless we have a
consensus on any change here, I think it is better to retain the
current text as it is.
I would like to close the Open item listed corresponding to this
thread [1] as the fix for the reported issue is committed
(fd0b9dcebd). Do let me know if you or others think otherwise?
[1] - https://wiki.postgresql.org/wiki/PostgreSQL_15_Open_Items
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: bogus: logical replication rows/cols combinations
@ 2022-06-16 04:23 Amit Kapila <[email protected]>
parent: Amit Kapila <[email protected]>
0 siblings, 0 replies; 58+ messages in thread
From: Amit Kapila @ 2022-06-16 04:23 UTC (permalink / raw)
To: Peter Smith <[email protected]>; +Cc: Justin Pryzby <[email protected]>; Hou, Zhijie/侯 志杰 <[email protected]>; Alvaro Herrera <[email protected]>; Tomas Vondra <[email protected]>; Peter Eisentraut <[email protected]>; PostgreSQL Hackers <[email protected]>; Tom Lane <[email protected]>
On Mon, Jun 13, 2022 at 8:54 AM Amit Kapila <[email protected]> wrote:
>
> I would like to close the Open item listed corresponding to this
> thread [1] as the fix for the reported issue is committed
> (fd0b9dcebd). Do let me know if you or others think otherwise?
>
Seeing no objections, I have closed this item.
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 58+ messages in thread
end of thread, other threads:[~2022-06-16 04:23 UTC | newest]
Thread overview: 58+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2020-11-11 12:51 [PATCH v3 1/2] In-place table persistence change Kyotaro Horiguchi <[email protected]>
2022-04-27 06:12 ` Re: bogus: logical replication rows/cols combinations Michael Paquier <[email protected]>
2022-04-28 15:35 ` Re: bogus: logical replication rows/cols combinations Tomas Vondra <[email protected]>
2022-05-02 16:30 ` Re: bogus: logical replication rows/cols combinations Alvaro Herrera <[email protected]>
2022-05-02 20:34 ` Re: bogus: logical replication rows/cols combinations Peter Eisentraut <[email protected]>
2022-05-03 19:40 ` Re: bogus: logical replication rows/cols combinations Tomas Vondra <[email protected]>
2022-05-04 13:56 ` Re: bogus: logical replication rows/cols combinations Peter Eisentraut <[email protected]>
2022-05-02 09:35 Re: bogus: logical replication rows/cols combinations Tomas Vondra <[email protected]>
2022-05-02 10:17 ` Re: bogus: logical replication rows/cols combinations Alvaro Herrera <[email protected]>
2022-05-02 10:23 ` Re: bogus: logical replication rows/cols combinations Tomas Vondra <[email protected]>
2022-05-02 10:55 ` Re: bogus: logical replication rows/cols combinations Alvaro Herrera <[email protected]>
2022-05-02 11:14 ` Re: bogus: logical replication rows/cols combinations Amit Kapila <[email protected]>
2022-05-02 11:44 ` Re: bogus: logical replication rows/cols combinations Alvaro Herrera <[email protected]>
2022-05-02 17:36 ` Re: bogus: logical replication rows/cols combinations Tomas Vondra <[email protected]>
2022-05-02 17:51 ` Re: bogus: logical replication rows/cols combinations Alvaro Herrera <[email protected]>
2022-05-02 18:40 ` Re: bogus: logical replication rows/cols combinations Tomas Vondra <[email protected]>
2022-05-03 03:30 ` Re: bogus: logical replication rows/cols combinations Amit Kapila <[email protected]>
2022-05-06 03:23 ` RE: bogus: logical replication rows/cols combinations [email protected] <[email protected]>
2022-05-06 12:26 ` Re: bogus: logical replication rows/cols combinations Tomas Vondra <[email protected]>
2022-05-06 13:40 ` Re: bogus: logical replication rows/cols combinations Amit Kapila <[email protected]>
2022-05-06 13:57 ` Re: bogus: logical replication rows/cols combinations Tomas Vondra <[email protected]>
2022-05-03 03:53 ` Re: bogus: logical replication rows/cols combinations Amit Kapila <[email protected]>
2022-05-07 05:36 ` Re: bogus: logical replication rows/cols combinations Amit Kapila <[email protected]>
2022-05-08 18:11 ` Re: bogus: logical replication rows/cols combinations Tomas Vondra <[email protected]>
2022-05-09 03:45 ` Re: bogus: logical replication rows/cols combinations Amit Kapila <[email protected]>
2022-05-10 19:05 ` Re: bogus: logical replication rows/cols combinations Tomas Vondra <[email protected]>
2022-05-11 03:33 ` Re: bogus: logical replication rows/cols combinations Amit Kapila <[email protected]>
2022-05-11 07:25 ` RE: bogus: logical replication rows/cols combinations [email protected] <[email protected]>
2022-05-12 06:45 ` Re: bogus: logical replication rows/cols combinations Amit Kapila <[email protected]>
2022-05-12 08:32 ` Re: bogus: logical replication rows/cols combinations Amit Kapila <[email protected]>
2022-05-13 06:02 ` RE: bogus: logical replication rows/cols combinations [email protected] <[email protected]>
2022-05-16 06:10 ` Re: bogus: logical replication rows/cols combinations Amit Kapila <[email protected]>
2022-05-16 12:34 ` RE: bogus: logical replication rows/cols combinations [email protected] <[email protected]>
2022-05-17 03:25 ` RE: bogus: logical replication rows/cols combinations [email protected] <[email protected]>
2022-05-17 06:49 ` RE: bogus: logical replication rows/cols combinations [email protected] <[email protected]>
2022-05-17 06:52 ` Re: bogus: logical replication rows/cols combinations Amit Kapila <[email protected]>
2022-05-17 09:10 ` RE: bogus: logical replication rows/cols combinations [email protected] <[email protected]>
2022-05-18 02:28 ` Re: bogus: logical replication rows/cols combinations Amit Kapila <[email protected]>
2022-05-16 13:20 ` Re: bogus: logical replication rows/cols combinations Alvaro Herrera <[email protected]>
2022-05-17 03:26 ` Re: bogus: logical replication rows/cols combinations Amit Kapila <[email protected]>
2022-05-19 05:03 ` Re: bogus: logical replication rows/cols combinations Amit Kapila <[email protected]>
2022-05-19 12:07 ` Re: bogus: logical replication rows/cols combinations Justin Pryzby <[email protected]>
2022-05-19 14:24 ` Re: bogus: logical replication rows/cols combinations Tom Lane <[email protected]>
2022-05-20 03:06 ` Re: bogus: logical replication rows/cols combinations Amit Kapila <[email protected]>
2022-05-24 05:33 ` RE: bogus: logical replication rows/cols combinations [email protected] <[email protected]>
2022-05-27 05:47 ` Re: bogus: logical replication rows/cols combinations Amit Kapila <[email protected]>
2022-05-27 05:53 ` Re: bogus: logical replication rows/cols combinations Justin Pryzby <[email protected]>
2022-05-27 07:34 ` RE: bogus: logical replication rows/cols combinations [email protected] <[email protected]>
2022-06-02 11:58 ` Re: bogus: logical replication rows/cols combinations Amit Kapila <[email protected]>
2022-06-06 05:42 ` Re: bogus: logical replication rows/cols combinations Peter Smith <[email protected]>
2022-06-08 03:25 ` Re: bogus: logical replication rows/cols combinations Justin Pryzby <[email protected]>
2022-06-08 05:35 ` Re: bogus: logical replication rows/cols combinations Peter Smith <[email protected]>
2022-06-13 03:24 ` Re: bogus: logical replication rows/cols combinations Amit Kapila <[email protected]>
2022-06-16 04:23 ` Re: bogus: logical replication rows/cols combinations Amit Kapila <[email protected]>
2022-05-24 09:49 ` Re: bogus: logical replication rows/cols combinations Amit Kapila <[email protected]>
2022-05-26 03:26 ` Re: bogus: logical replication rows/cols combinations Amit Kapila <[email protected]>
2022-05-02 11:23 ` Re: bogus: logical replication rows/cols combinations Amit Kapila <[email protected]>
2022-05-02 18:37 ` Re: bogus: logical replication rows/cols combinations Tomas Vondra <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox