public inbox for [email protected]  

help / color / mirror / Atom feed

[PATCH v8 4/8] Propagate changes to indisclustered to child/parents
9+ messages / 3 participants
[nested] [flat]

* [PATCH v8 4/8] Propagate changes to indisclustered to child/parents
@ 2020-10-07 03:11  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 9+ messages in thread

From: Justin Pryzby @ 2020-10-07 03:11 UTC (permalink / raw)

---
 src/backend/commands/cluster.c        | 109 ++++++++++++++++----------
 src/backend/commands/indexcmds.c      |   2 +
 src/test/regress/expected/cluster.out |  46 +++++++++++
 src/test/regress/sql/cluster.sql      |  11 +++
 4 files changed, 125 insertions(+), 43 deletions(-)

diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index cb4fc350c6..5c08f0642e 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -74,6 +74,7 @@ static void rebuild_relation(Relation OldHeap, Oid indexOid, bool verbose);
 static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 							bool verbose, bool *pSwapToastByContent,
 							TransactionId *pFreezeXid, MultiXactId *pCutoffMulti);
+static void set_indisclustered(Oid indexOid, bool isclustered, Relation pg_index);
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
@@ -516,66 +517,88 @@ check_index_is_clusterable(Relation OldHeap, Oid indexOid, bool recheck, LOCKMOD
 	index_close(OldIndex, NoLock);
 }
 
+/*
+ * Helper for mark_index_clustered
+ * Mark a single index as clustered or not.
+ * pg_index is passed by caller to avoid repeatedly re-opening it.
+ */
+static void
+set_indisclustered(Oid indexOid, bool isclustered, Relation pg_index)
+{
+	HeapTuple	indexTuple;
+	Form_pg_index indexForm;
+
+	indexTuple = SearchSysCacheCopy1(INDEXRELID,
+									ObjectIdGetDatum(indexOid));
+	if (!HeapTupleIsValid(indexTuple))
+		elog(ERROR, "cache lookup failed for index %u", indexOid);
+	indexForm = (Form_pg_index) GETSTRUCT(indexTuple);
+
+	/* this was checked earlier, but let's be real sure */
+	if (isclustered && !indexForm->indisvalid)
+		elog(ERROR, "cannot cluster on invalid index %u", indexOid);
+
+	indexForm->indisclustered = isclustered;
+	CatalogTupleUpdate(pg_index, &indexTuple->t_self, indexTuple);
+	heap_freetuple(indexTuple);
+}
+
 /*
  * mark_index_clustered: mark the specified index as the one clustered on
  *
- * With indexOid == InvalidOid, will mark all indexes of rel not-clustered.
+ * With indexOid == InvalidOid, mark all indexes of rel not-clustered.
+ * Otherwise, mark children of the clustered index as clustered, and parents of
+ * other indexes as unclustered.
+ * We wish to maintain the following properties:
+ * 1) Only one index on a relation can be marked clustered at once
+ * 2) If a partitioned index is clustered, then all its children must be
+ *     clustered.
  */
 void
 mark_index_clustered(Relation rel, Oid indexOid, bool is_internal)
 {
-	HeapTuple	indexTuple;
-	Form_pg_index indexForm;
-	Relation	pg_index;
-	ListCell   *index;
-
-	/*
-	 * If the index is already marked clustered, no need to do anything.
-	 */
-	if (OidIsValid(indexOid))
-	{
-		if (get_index_isclustered(indexOid))
-			return;
-	}
+	ListCell	*lc, *lc2;
+	List		*indexes;
+	Relation	pg_index = table_open(IndexRelationId, RowExclusiveLock);
+	List		*inh = find_all_inheritors(RelationGetRelid(rel), ShareRowExclusiveLock, NULL);
 
 	/*
 	 * Check each index of the relation and set/clear the bit as needed.
+	 * Iterate over the relation's children rather than the index's children
+	 * since we need to unset cluster for indexes on intermediate children,
+	 * too.
 	 */
-	pg_index = table_open(IndexRelationId, RowExclusiveLock);
-
-	foreach(index, RelationGetIndexList(rel))
+	foreach(lc, inh)
 	{
-		Oid			thisIndexOid = lfirst_oid(index);
-
-		indexTuple = SearchSysCacheCopy1(INDEXRELID,
-										 ObjectIdGetDatum(thisIndexOid));
-		if (!HeapTupleIsValid(indexTuple))
-			elog(ERROR, "cache lookup failed for index %u", thisIndexOid);
-		indexForm = (Form_pg_index) GETSTRUCT(indexTuple);
+		Oid			inhrelid = lfirst_oid(lc);
+		Relation	thisrel = table_open(inhrelid, ShareRowExclusiveLock);
 
-		/*
-		 * Unset the bit if set.  We know it's wrong because we checked this
-		 * earlier.
-		 */
-		if (indexForm->indisclustered)
+		indexes = RelationGetIndexList(thisrel);
+		foreach (lc2, indexes)
 		{
-			indexForm->indisclustered = false;
-			CatalogTupleUpdate(pg_index, &indexTuple->t_self, indexTuple);
-		}
-		else if (thisIndexOid == indexOid)
-		{
-			/* this was checked earlier, but let's be real sure */
-			if (!indexForm->indisvalid)
-				elog(ERROR, "cannot cluster on invalid index %u", indexOid);
-			indexForm->indisclustered = true;
-			CatalogTupleUpdate(pg_index, &indexTuple->t_self, indexTuple);
-		}
+			bool	isclustered;
+			Oid		thisIndexOid = lfirst_oid(lc2);
+			List	*parentoids = get_rel_relispartition(thisIndexOid) ?
+				get_partition_ancestors(thisIndexOid) : NIL;
 
-		InvokeObjectPostAlterHookArg(IndexRelationId, thisIndexOid, 0,
-									 InvalidOid, is_internal);
+			/*
+			 * A child of the clustered index must be set clustered;
+			 * indexes which are not children of the clustered index are
+			 * set unclustered
+			 */
+			isclustered = (thisIndexOid == indexOid) ||
+					list_member_oid(parentoids, indexOid);
+			Assert(OidIsValid(indexOid) || !isclustered);
+			set_indisclustered(thisIndexOid, isclustered, pg_index);
+
+			InvokeObjectPostAlterHookArg(IndexRelationId, thisIndexOid, 0,
+										 InvalidOid, is_internal);
+		}
 
-		heap_freetuple(indexTuple);
+		list_free(indexes);
+		table_close(thisrel, ShareRowExclusiveLock);
 	}
+	list_free(inh);
 
 	table_close(pg_index, RowExclusiveLock);
 }
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 127ba7835d..4ca1ffbfa4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -25,6 +25,7 @@
 #include "catalog/catalog.h"
 #include "catalog/index.h"
 #include "catalog/indexing.h"
+#include "catalog/partition.h"
 #include "catalog/pg_am.h"
 #include "catalog/pg_constraint.h"
 #include "catalog/pg_inherits.h"
@@ -32,6 +33,7 @@
 #include "catalog/pg_opfamily.h"
 #include "catalog/pg_tablespace.h"
 #include "catalog/pg_type.h"
+#include "commands/cluster.h"
 #include "commands/comment.h"
 #include "commands/dbcommands.h"
 #include "commands/defrem.h"
diff --git a/src/test/regress/expected/cluster.out b/src/test/regress/expected/cluster.out
index c74cfa88cc..1d436dfaae 100644
--- a/src/test/regress/expected/cluster.out
+++ b/src/test/regress/expected/cluster.out
@@ -495,6 +495,52 @@ Indexes:
     "clstrpart_idx" btree (a) CLUSTER
 Number of partitions: 3 (Use \d+ to list them.)
 
+-- Test that it recurses to grandchildren:
+\d clstrpart33
+            Table "public.clstrpart33"
+ Column |  Type   | Collation | Nullable | Default 
+--------+---------+-----------+----------+---------
+ a      | integer |           |          | 
+Partition of: clstrpart3 DEFAULT
+Indexes:
+    "clstrpart33_a_idx" btree (a) CLUSTER
+
+ALTER TABLE clstrpart SET WITHOUT CLUSTER;
+\d clstrpart33
+            Table "public.clstrpart33"
+ Column |  Type   | Collation | Nullable | Default 
+--------+---------+-----------+----------+---------
+ a      | integer |           |          | 
+Partition of: clstrpart3 DEFAULT
+Indexes:
+    "clstrpart33_a_idx" btree (a)
+
+ALTER TABLE clstrpart CLUSTER ON clstrpart_idx;
+\d clstrpart33
+            Table "public.clstrpart33"
+ Column |  Type   | Collation | Nullable | Default 
+--------+---------+-----------+----------+---------
+ a      | integer |           |          | 
+Partition of: clstrpart3 DEFAULT
+Indexes:
+    "clstrpart33_a_idx" btree (a) CLUSTER
+
+-- Check that only one child is marked clustered after marking clustered on a different parent
+CREATE INDEX clstrpart1_idx_2 ON clstrpart1(a);
+ALTER TABLE clstrpart CLUSTER ON clstrpart_idx;
+ALTER TABLE clstrpart1 CLUSTER ON clstrpart1_idx_2;
+\d clstrpart1
+       Partitioned table "public.clstrpart1"
+ Column |  Type   | Collation | Nullable | Default 
+--------+---------+-----------+----------+---------
+ a      | integer |           |          | 
+Partition of: clstrpart FOR VALUES FROM (1) TO (10)
+Partition key: RANGE (a)
+Indexes:
+    "clstrpart1_a_idx" btree (a)
+    "clstrpart1_idx_2" btree (a) CLUSTER
+Number of partitions: 2 (Use \d+ to list them.)
+
 -- Test CLUSTER with external tuplesorting
 create table clstr_4 as select * from tenk1;
 create index cluster_sort on clstr_4 (hundred, thousand, tenthous);
diff --git a/src/test/regress/sql/cluster.sql b/src/test/regress/sql/cluster.sql
index 9bcc77695c..0ded2be1ca 100644
--- a/src/test/regress/sql/cluster.sql
+++ b/src/test/regress/sql/cluster.sql
@@ -220,6 +220,17 @@ CLUSTER clstrpart1 USING clstrpart1_a_idx; -- partition which is itself partitio
 CLUSTER clstrpart12 USING clstrpart12_a_idx; -- partition which is itself partitioned, no childs
 CLUSTER clstrpart2 USING clstrpart2_a_idx; -- leaf
 \d clstrpart
+-- Test that it recurses to grandchildren:
+\d clstrpart33
+ALTER TABLE clstrpart SET WITHOUT CLUSTER;
+\d clstrpart33
+ALTER TABLE clstrpart CLUSTER ON clstrpart_idx;
+\d clstrpart33
+-- Check that only one child is marked clustered after marking clustered on a different parent
+CREATE INDEX clstrpart1_idx_2 ON clstrpart1(a);
+ALTER TABLE clstrpart CLUSTER ON clstrpart_idx;
+ALTER TABLE clstrpart1 CLUSTER ON clstrpart1_idx_2;
+\d clstrpart1
 
 -- Test CLUSTER with external tuplesorting
 
-- 
2.17.0


--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v8-0005-Invalidate-parent-indexes.patch"



^ permalink  raw  reply  [nested|flat] 9+ messages in thread

* Re: Avoid orphaned objects dependencies, take 3
@ 2024-04-22 10:52  Bertrand Drouvot <[email protected]>
  0 siblings, 1 reply; 9+ messages in thread

From: Bertrand Drouvot @ 2024-04-22 10:52 UTC (permalink / raw)
  To: Alexander Lakhin <[email protected]>; +Cc: [email protected]

Hi,

On Mon, Apr 22, 2024 at 01:00:00PM +0300, Alexander Lakhin wrote:
> Hi Bertrand,
> 
> 22.04.2024 11:45, Bertrand Drouvot wrote:
> > Hi,
> > 
> > This new thread is a follow-up of [1] and [2].
> > 
> > Problem description:
> > 
> > We have occasionally observed objects having an orphaned dependency, the
> > most common case we have seen is functions not linked to any namespaces.
> > 
> > ...
> > 
> > Looking forward to your feedback,
> 
> This have reminded me of bug #17182 [1].

Thanks for the link to the bug!

> Unfortunately, with the patch applied, the following script:
> 
> for ((i=1;i<=100;i++)); do
>   ( { for ((n=1;n<=20;n++)); do echo "DROP SCHEMA s;"; done } | psql ) >psql1.log 2>&1 &
>   echo "
> CREATE SCHEMA s;
> CREATE FUNCTION s.func1() RETURNS int LANGUAGE SQL AS 'SELECT 1;';
> CREATE FUNCTION s.func2() RETURNS int LANGUAGE SQL AS 'SELECT 2;';
> CREATE FUNCTION s.func3() RETURNS int LANGUAGE SQL AS 'SELECT 3;';
> CREATE FUNCTION s.func4() RETURNS int LANGUAGE SQL AS 'SELECT 4;';
> CREATE FUNCTION s.func5() RETURNS int LANGUAGE SQL AS 'SELECT 5;';
>   "  | psql >psql2.log 2>&1 &
>   wait
>   psql -c "DROP SCHEMA s CASCADE" >psql3.log
> done
> echo "
> SELECT pg_identify_object('pg_proc'::regclass, pp.oid, 0), pp.oid FROM pg_proc pp
>   LEFT JOIN pg_namespace pn ON pp.pronamespace = pn.oid WHERE pn.oid IS NULL" | psql
> 
> still ends with:
> server closed the connection unexpectedly
>         This probably means the server terminated abnormally
>         before or while processing the request.
> 
> 2024-04-22 09:54:39.171 UTC|||662633dc.152bbc|LOG:  server process (PID
> 1388378) was terminated by signal 11: Segmentation fault
> 2024-04-22 09:54:39.171 UTC|||662633dc.152bbc|DETAIL:  Failed process was
> running: SELECT pg_identify_object('pg_proc'::regclass, pp.oid, 0), pp.oid
> FROM pg_proc pp
>       LEFT JOIN pg_namespace pn ON pp.pronamespace = pn.oid WHERE pn.oid IS NULL
> 

Thanks for sharing the script.

That's weird, I just launched it several times with the patch applied and I'm not
able to see the seg fault (while I can see it constently failing on the master
branch).

Are you 100% sure you tested it against a binary with the patch applied?

Regards,

-- 
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com






^ permalink  raw  reply  [nested|flat] 9+ messages in thread

* Re: Avoid orphaned objects dependencies, take 3
@ 2024-04-22 12:00  Alexander Lakhin <[email protected]>
  parent: Bertrand Drouvot <[email protected]>
  0 siblings, 1 reply; 9+ messages in thread

From: Alexander Lakhin @ 2024-04-22 12:00 UTC (permalink / raw)
  To: Bertrand Drouvot <[email protected]>; +Cc: [email protected]

22.04.2024 13:52, Bertrand Drouvot wrote:
>
> That's weird, I just launched it several times with the patch applied and I'm not
> able to see the seg fault (while I can see it constently failing on the master
> branch).
>
> Are you 100% sure you tested it against a binary with the patch applied?
>

Yes, at least I can't see what I'm doing wrong. Please try my
self-contained script attached.

Best regards,
Alexander
#!/bin/bash

git reset --hard HEAD; git clean -dfx >/dev/null
wget https://www.postgresql.org/message-id/attachment/159685/v1-0001-Avoid-orphaned-objects-dependencies....
git apply v1-0001-Avoid-orphaned-objects-dependencies.patch || exit 1
CPPFLAGS="-Og" ./configure --enable-debug --enable-cassert -q && make -j8 -s && TESTS="test_setup" make -s check-tests

PGROOT=`pwd`/tmp_install
export PGDATA=`pwd`/tmpdb

export PATH="$PGROOT/usr/local/pgsql/bin:$PATH"
export LD_LIBRARY_PATH="$PGROOT/usr/local/pgsql/lib"

initdb >initdb.log 2>&1

export PGPORT=55432
echo "
port=$PGPORT
fsync = off
" > $PGDATA/postgresql.auto.conf

pg_ctl -l server.log start
export PGDATABASE=regression

createdb regression

for ((i=1;i<=300;i++)); do
  ( { for ((n=1;n<=20;n++)); do echo "DROP SCHEMA s;"; done } | psql ) >psql1.log 2>&1 & 
  echo "
CREATE SCHEMA s;
CREATE FUNCTION s.func1() RETURNS int LANGUAGE SQL AS 'SELECT 1;';
CREATE FUNCTION s.func2() RETURNS int LANGUAGE SQL AS 'SELECT 2;';
CREATE FUNCTION s.func3() RETURNS int LANGUAGE SQL AS 'SELECT 3;';
CREATE FUNCTION s.func4() RETURNS int LANGUAGE SQL AS 'SELECT 4;';
CREATE FUNCTION s.func5() RETURNS int LANGUAGE SQL AS 'SELECT 5;';
  "  | psql >psql2.log 2>&1 &
  wait
  psql -c "DROP SCHEMA s CASCADE" >psql3.log
done
echo "
SELECT pg_identify_object('pg_proc'::regclass, pp.oid, 0), pp.oid FROM pg_proc pp
  LEFT JOIN pg_namespace pn ON pp.pronamespace = pn.oid WHERE pn.oid IS NULL" | psql

pg_ctl -w -t 5 -D "$PGDATA" stop


Attachments:

  [text/plain] repro-17182.sh.txt (1.4K, ../../[email protected]/2-repro-17182.sh.txt)
  download | inline:
#!/bin/bash

git reset --hard HEAD; git clean -dfx >/dev/null
wget https://www.postgresql.org/message-id/attachment/159685/v1-0001-Avoid-orphaned-objects-dependencies.patch
git apply v1-0001-Avoid-orphaned-objects-dependencies.patch || exit 1
CPPFLAGS="-Og" ./configure --enable-debug --enable-cassert -q && make -j8 -s && TESTS="test_setup" make -s check-tests

PGROOT=`pwd`/tmp_install
export PGDATA=`pwd`/tmpdb

export PATH="$PGROOT/usr/local/pgsql/bin:$PATH"
export LD_LIBRARY_PATH="$PGROOT/usr/local/pgsql/lib"

initdb >initdb.log 2>&1

export PGPORT=55432
echo "
port=$PGPORT
fsync = off
" > $PGDATA/postgresql.auto.conf

pg_ctl -l server.log start
export PGDATABASE=regression

createdb regression

for ((i=1;i<=300;i++)); do
  ( { for ((n=1;n<=20;n++)); do echo "DROP SCHEMA s;"; done } | psql ) >psql1.log 2>&1 & 
  echo "
CREATE SCHEMA s;
CREATE FUNCTION s.func1() RETURNS int LANGUAGE SQL AS 'SELECT 1;';
CREATE FUNCTION s.func2() RETURNS int LANGUAGE SQL AS 'SELECT 2;';
CREATE FUNCTION s.func3() RETURNS int LANGUAGE SQL AS 'SELECT 3;';
CREATE FUNCTION s.func4() RETURNS int LANGUAGE SQL AS 'SELECT 4;';
CREATE FUNCTION s.func5() RETURNS int LANGUAGE SQL AS 'SELECT 5;';
  "  | psql >psql2.log 2>&1 &
  wait
  psql -c "DROP SCHEMA s CASCADE" >psql3.log
done
echo "
SELECT pg_identify_object('pg_proc'::regclass, pp.oid, 0), pp.oid FROM pg_proc pp
  LEFT JOIN pg_namespace pn ON pp.pronamespace = pn.oid WHERE pn.oid IS NULL" | psql

pg_ctl -w -t 5 -D "$PGDATA" stop

^ permalink  raw  reply  [nested|flat] 9+ messages in thread

* Re: Avoid orphaned objects dependencies, take 3
@ 2024-04-23 04:59  Bertrand Drouvot <[email protected]>
  parent: Alexander Lakhin <[email protected]>
  0 siblings, 1 reply; 9+ messages in thread

From: Bertrand Drouvot @ 2024-04-23 04:59 UTC (permalink / raw)
  To: Alexander Lakhin <[email protected]>; +Cc: [email protected]

Hi,

On Mon, Apr 22, 2024 at 03:00:00PM +0300, Alexander Lakhin wrote:
> 22.04.2024 13:52, Bertrand Drouvot wrote:
> > 
> > That's weird, I just launched it several times with the patch applied and I'm not
> > able to see the seg fault (while I can see it constently failing on the master
> > branch).
> > 
> > Are you 100% sure you tested it against a binary with the patch applied?
> > 
> 
> Yes, at least I can't see what I'm doing wrong. Please try my
> self-contained script attached.

Thanks for sharing your script!

Yeah your script ensures the patch is applied before the repro is executed.

I do confirm that I can also see the issue with the patch applied (but I had to
launch multiple attempts, while on master one attempt is enough).

I'll have a look.

Regards,

-- 
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com






^ permalink  raw  reply  [nested|flat] 9+ messages in thread

* Re: Avoid orphaned objects dependencies, take 3
@ 2024-04-23 16:20  Bertrand Drouvot <[email protected]>
  parent: Bertrand Drouvot <[email protected]>
  0 siblings, 1 reply; 9+ messages in thread

From: Bertrand Drouvot @ 2024-04-23 16:20 UTC (permalink / raw)
  To: Alexander Lakhin <[email protected]>; +Cc: [email protected]

Hi,

On Tue, Apr 23, 2024 at 04:59:09AM +0000, Bertrand Drouvot wrote:
> Hi,
> 
> On Mon, Apr 22, 2024 at 03:00:00PM +0300, Alexander Lakhin wrote:
> > 22.04.2024 13:52, Bertrand Drouvot wrote:
> > > 
> > > That's weird, I just launched it several times with the patch applied and I'm not
> > > able to see the seg fault (while I can see it constently failing on the master
> > > branch).
> > > 
> > > Are you 100% sure you tested it against a binary with the patch applied?
> > > 
> > 
> > Yes, at least I can't see what I'm doing wrong. Please try my
> > self-contained script attached.
> 
> Thanks for sharing your script!
> 
> Yeah your script ensures the patch is applied before the repro is executed.
> 
> I do confirm that I can also see the issue with the patch applied (but I had to
> launch multiple attempts, while on master one attempt is enough).
> 
> I'll have a look.

Please find attached v2 that should not produce the issue anymore (I launched a
lot of attempts without any issues). v1 was not strong enough as it was not
always checking for the dependent object existence. v2 now always checks if the
object still exists after the additional lock acquisition attempt while recording
the dependency.

I still need to think about v2 but in the meantime could you please also give
v2 a try on you side?

Regards,

-- 
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com


Attachments:

  [text/x-diff] v2-0001-Avoid-orphaned-objects-dependencies.patch (14.2K, ../../[email protected]/2-v2-0001-Avoid-orphaned-objects-dependencies.patch)
  download | inline diff:
From f80fdfc32e791463555aa318f26ff5e7363ac3ac Mon Sep 17 00:00:00 2001
From: Bertrand Drouvot <[email protected]>
Date: Fri, 29 Mar 2024 15:43:26 +0000
Subject: [PATCH v2] Avoid orphaned objects dependencies

It's currently possible to create orphaned objects dependencies, for example:

Scenario 1:

session 1: begin; drop schema schem;
session 2: create a function in the schema schem
session 1: commit;

With the above, the function created in session 2 would be linked to a non
existing schema.

Scenario 2:

session 1: begin; create a function in the schema schem
session 2: drop schema schem;
session 1: commit;

With the above, the function created in session 1 would be linked to a non
existing schema.

To avoid those scenarios, a new lock (that conflicts with a lock taken by DROP)
has been put in place when the dependencies are being recorded. With this in
place, the drop schema in scenario 2 would be locked.

Also, after locking the object, the patch checks that the object still exists:
with this in place session 2 in scenario 1 would be locked and would report an
error once session 1 committs (that would not be the case should session 1 abort
the transaction).

The patch adds a few tests for some dependency cases (that would currently produce
orphaned objects):

- schema and function (as the above scenarios)
- function and type
- table and type
---
 src/backend/catalog/dependency.c              | 48 +++++++++++++
 src/backend/catalog/objectaddress.c           | 70 +++++++++++++++++++
 src/backend/catalog/pg_depend.c               |  6 ++
 src/include/catalog/dependency.h              |  1 +
 src/include/catalog/objectaddress.h           |  1 +
 src/test/modules/Makefile                     |  1 +
 src/test/modules/meson.build                  |  1 +
 .../test_dependencies_locks/.gitignore        |  3 +
 .../modules/test_dependencies_locks/Makefile  | 14 ++++
 .../expected/test_dependencies_locks.out      | 49 +++++++++++++
 .../test_dependencies_locks/meson.build       | 12 ++++
 .../specs/test_dependencies_locks.spec        | 39 +++++++++++
 12 files changed, 245 insertions(+)
  38.1% src/backend/catalog/
  30.7% src/test/modules/test_dependencies_locks/expected/
  19.5% src/test/modules/test_dependencies_locks/specs/
   8.7% src/test/modules/test_dependencies_locks/

diff --git a/src/backend/catalog/dependency.c b/src/backend/catalog/dependency.c
index d4b5b2ade1..e3b66025dd 100644
--- a/src/backend/catalog/dependency.c
+++ b/src/backend/catalog/dependency.c
@@ -1517,6 +1517,54 @@ AcquireDeletionLock(const ObjectAddress *object, int flags)
 	}
 }
 
+/*
+ * depLockAndCheckObject
+ *
+ * Lock the object that we are about to record a dependency on.
+ * After it's locked, verify that it hasn't been dropped while we
+ * weren't looking.  If the object has been dropped, this function
+ * does not return!
+ */
+void
+depLockAndCheckObject(const ObjectAddress *object)
+{
+	char	   *object_description;
+
+	/*
+	 * Those don't rely on LockDatabaseObject() when being dropped (see
+	 * AcquireDeletionLock()). Also it looks like they can not produce
+	 * orphaned dependent objects when being dropped.
+	 */
+	if (object->classId == RelationRelationId || object->classId == AuthMemRelationId)
+		return;
+
+	object_description = getObjectDescription(object, true);
+
+	/* assume we should lock the whole object not a sub-object */
+	LockDatabaseObject(object->classId, object->objectId, 0, AccessShareLock);
+
+	/* check if object still exists */
+	if (!ObjectByIdExist(object, false))
+	{
+		/*
+		 * It might be possible that we are creating it, so bypass the syscache
+		 * lookup and use a dirty snaphot instead.
+		 */
+		if (!ObjectByIdExist(object, true))
+		{
+			if(object_description)
+				ereport(ERROR, errmsg("%s does not exist", object_description));
+			else
+				ereport(ERROR, errmsg("a dependent object does not exist"));
+		}
+	}
+
+	if (object_description)
+		pfree(object_description);
+
+	return;
+}
+
 /*
  * ReleaseDeletionLock - release an object deletion lock
  *
diff --git a/src/backend/catalog/objectaddress.c b/src/backend/catalog/objectaddress.c
index 7b536ac6fd..c2b873dd81 100644
--- a/src/backend/catalog/objectaddress.c
+++ b/src/backend/catalog/objectaddress.c
@@ -2590,6 +2590,76 @@ get_object_namespace(const ObjectAddress *address)
 	return oid;
 }
 
+/*
+ * ObjectByIdExist
+ *
+ * Return whether the given object exists.
+ *
+ * Works for most catalogs, if no special processing is needed.
+ */
+bool
+ObjectByIdExist(const ObjectAddress *address, bool use_dirty_snapshot)
+{
+	HeapTuple	tuple;
+	int			cache = -1;
+	const ObjectPropertyType *property;
+
+	if (!use_dirty_snapshot)
+	{
+		property = get_object_property_data(address->classId);
+
+		cache = property->oid_catcache_id;
+	}
+
+	if (cache >= 0)
+	{
+		/* Fetch tuple from syscache. */
+		tuple = SearchSysCache1(cache, ObjectIdGetDatum(address->objectId));
+
+		if (!HeapTupleIsValid(tuple))
+		{
+			return false;
+		}
+
+		ReleaseSysCache(tuple);
+
+		return true;
+	}
+	else
+	{
+		Relation	rel;
+		ScanKeyData skey[1];
+		SysScanDesc scan;
+		SnapshotData DirtySnapshot;
+		Snapshot	snapshot;
+
+		if (use_dirty_snapshot)
+		{
+			InitDirtySnapshot(DirtySnapshot);
+			snapshot = &DirtySnapshot;
+		}
+		else
+			snapshot = NULL;
+
+		rel = table_open(address->classId, AccessShareLock);
+
+		ScanKeyInit(&skey[0],
+					get_object_attnum_oid(address->classId),
+					BTEqualStrategyNumber, F_OIDEQ,
+					ObjectIdGetDatum(address->objectId));
+
+		scan = systable_beginscan(rel, get_object_oid_index(address->classId), true,
+								  snapshot, 1, skey);
+
+		/* we expect exactly one match */
+		tuple = systable_getnext(scan);
+		systable_endscan(scan);
+		table_close(rel, AccessShareLock);
+
+		return (HeapTupleIsValid(tuple));
+	}
+}
+
 /*
  * Return ObjectType for the given object type as given by
  * getObjectTypeDescription; if no valid ObjectType code exists, but it's a
diff --git a/src/backend/catalog/pg_depend.c b/src/backend/catalog/pg_depend.c
index f85a898de8..6bb218f5dd 100644
--- a/src/backend/catalog/pg_depend.c
+++ b/src/backend/catalog/pg_depend.c
@@ -106,6 +106,12 @@ recordMultipleDependencies(const ObjectAddress *depender,
 		if (isObjectPinned(referenced))
 			continue;
 
+		/*
+		 * Acquire a lock and check object still exists while recording the
+		 * dependency. XXX - Should we do so only for DEPENDENCY_NORMAL?
+		 */
+		depLockAndCheckObject(referenced);
+
 		if (slot_init_count < max_slots)
 		{
 			slot[slot_stored_count] = MakeSingleTupleTableSlot(RelationGetDescr(dependDesc),
diff --git a/src/include/catalog/dependency.h b/src/include/catalog/dependency.h
index ec654010d4..8915548711 100644
--- a/src/include/catalog/dependency.h
+++ b/src/include/catalog/dependency.h
@@ -94,6 +94,7 @@ typedef struct ObjectAddresses ObjectAddresses;
 /* in dependency.c */
 
 extern void AcquireDeletionLock(const ObjectAddress *object, int flags);
+extern void depLockAndCheckObject(const ObjectAddress *object);
 
 extern void ReleaseDeletionLock(const ObjectAddress *object);
 
diff --git a/src/include/catalog/objectaddress.h b/src/include/catalog/objectaddress.h
index 3a70d80e32..04891abcc1 100644
--- a/src/include/catalog/objectaddress.h
+++ b/src/include/catalog/objectaddress.h
@@ -53,6 +53,7 @@ extern void check_object_ownership(Oid roleid,
 								   Node *object, Relation relation);
 
 extern Oid	get_object_namespace(const ObjectAddress *address);
+extern bool ObjectByIdExist(const ObjectAddress *address, bool use_dirty_snapshot);
 
 extern bool is_objectclass_supported(Oid class_id);
 extern const char *get_object_class_descr(Oid class_id);
diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile
index 256799f520..75f357100f 100644
--- a/src/test/modules/Makefile
+++ b/src/test/modules/Makefile
@@ -17,6 +17,7 @@ SUBDIRS = \
 		  test_copy_callbacks \
 		  test_custom_rmgrs \
 		  test_ddl_deparse \
+		  test_dependencies_locks \
 		  test_dsa \
 		  test_dsm_registry \
 		  test_extensions \
diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build
index d8fe059d23..60305dcccd 100644
--- a/src/test/modules/meson.build
+++ b/src/test/modules/meson.build
@@ -16,6 +16,7 @@ subdir('test_bloomfilter')
 subdir('test_copy_callbacks')
 subdir('test_custom_rmgrs')
 subdir('test_ddl_deparse')
+subdir('test_dependencies_locks')
 subdir('test_dsa')
 subdir('test_dsm_registry')
 subdir('test_extensions')
diff --git a/src/test/modules/test_dependencies_locks/.gitignore b/src/test/modules/test_dependencies_locks/.gitignore
new file mode 100644
index 0000000000..bf000faac4
--- /dev/null
+++ b/src/test/modules/test_dependencies_locks/.gitignore
@@ -0,0 +1,3 @@
+# Generated subdirectories
+/log/
+/output_iso
diff --git a/src/test/modules/test_dependencies_locks/Makefile b/src/test/modules/test_dependencies_locks/Makefile
new file mode 100644
index 0000000000..7491048380
--- /dev/null
+++ b/src/test/modules/test_dependencies_locks/Makefile
@@ -0,0 +1,14 @@
+# src/test/modules/test_dependencies_locks/Makefile
+
+ISOLATION = test_dependencies_locks
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = src/test/modules/test_dependencies_locks
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/src/test/modules/test_dependencies_locks/expected/test_dependencies_locks.out b/src/test/modules/test_dependencies_locks/expected/test_dependencies_locks.out
new file mode 100644
index 0000000000..d0980f77d5
--- /dev/null
+++ b/src/test/modules/test_dependencies_locks/expected/test_dependencies_locks.out
@@ -0,0 +1,49 @@
+Parsed test spec with 2 sessions
+
+starting permutation: s1_begin s1_create_function_in_schema s2_drop_schema s1_commit
+step s1_begin: BEGIN;
+step s1_create_function_in_schema: CREATE FUNCTION testschema.foo() RETURNS int AS 'select 1' LANGUAGE sql;
+step s2_drop_schema: DROP SCHEMA testschema; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_schema: <... completed>
+ERROR:  cannot drop schema testschema because other objects depend on it
+
+starting permutation: s2_begin s2_drop_schema s1_create_function_in_schema s2_commit
+step s2_begin: BEGIN;
+step s2_drop_schema: DROP SCHEMA testschema;
+step s1_create_function_in_schema: CREATE FUNCTION testschema.foo() RETURNS int AS 'select 1' LANGUAGE sql; <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_function_in_schema: <... completed>
+ERROR:  schema testschema does not exist
+
+starting permutation: s1_begin s1_create_function_with_type s2_drop_foo_type s1_commit
+step s1_begin: BEGIN;
+step s1_create_function_with_type: CREATE FUNCTION footype(num foo) RETURNS int AS 'select 1' LANGUAGE sql;
+step s2_drop_foo_type: DROP TYPE public.foo; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_foo_type: <... completed>
+ERROR:  cannot drop type foo because other objects depend on it
+
+starting permutation: s2_begin s2_drop_foo_type s1_create_function_with_type s2_commit
+step s2_begin: BEGIN;
+step s2_drop_foo_type: DROP TYPE public.foo;
+step s1_create_function_with_type: CREATE FUNCTION footype(num foo) RETURNS int AS 'select 1' LANGUAGE sql; <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_function_with_type: <... completed>
+ERROR:  type foo does not exist
+
+starting permutation: s1_begin s1_create_table_with_type s2_drop_footab_type s1_commit
+step s1_begin: BEGIN;
+step s1_create_table_with_type: CREATE TABLE tabtype(a footab);
+step s2_drop_footab_type: DROP TYPE public.footab; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_footab_type: <... completed>
+ERROR:  cannot drop type footab because other objects depend on it
+
+starting permutation: s2_begin s2_drop_footab_type s1_create_table_with_type s2_commit
+step s2_begin: BEGIN;
+step s2_drop_footab_type: DROP TYPE public.footab;
+step s1_create_table_with_type: CREATE TABLE tabtype(a footab); <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_table_with_type: <... completed>
+ERROR:  type footab does not exist
diff --git a/src/test/modules/test_dependencies_locks/meson.build b/src/test/modules/test_dependencies_locks/meson.build
new file mode 100644
index 0000000000..92a978ab93
--- /dev/null
+++ b/src/test/modules/test_dependencies_locks/meson.build
@@ -0,0 +1,12 @@
+# Copyright (c) 2024, PostgreSQL Global Development Group
+
+tests += {
+  'name': 'test_dependencies_locks',
+  'sd': meson.current_source_dir(),
+  'bd': meson.current_build_dir(),
+  'isolation': {
+    'specs': [
+      'test_dependencies_locks',
+    ],
+  },
+}
diff --git a/src/test/modules/test_dependencies_locks/specs/test_dependencies_locks.spec b/src/test/modules/test_dependencies_locks/specs/test_dependencies_locks.spec
new file mode 100644
index 0000000000..fd15bd2a78
--- /dev/null
+++ b/src/test/modules/test_dependencies_locks/specs/test_dependencies_locks.spec
@@ -0,0 +1,39 @@
+setup
+{
+  CREATE SCHEMA testschema;
+  CREATE TYPE public.foo as enum ('one', 'two');
+  CREATE TYPE public.footab as enum ('three', 'four');
+}
+
+teardown
+{
+  DROP FUNCTION IF EXISTS testschema.foo();
+  DROP FUNCTION IF EXISTS footype(num foo);
+  DROP TABLE IF EXISTS tabtype;
+  DROP SCHEMA IF EXISTS testschema;
+  DROP TYPE IF EXISTS public.foo;
+  DROP TYPE IF EXISTS public.footab;
+}
+
+session "s1"
+
+step "s1_begin" { BEGIN; }
+step "s1_create_function_in_schema" { CREATE FUNCTION testschema.foo() RETURNS int AS 'select 1' LANGUAGE sql; }
+step "s1_create_function_with_type" { CREATE FUNCTION footype(num foo) RETURNS int AS 'select 1' LANGUAGE sql; }
+step "s1_create_table_with_type" { CREATE TABLE tabtype(a footab); }
+step "s1_commit" { COMMIT; }
+
+session "s2"
+
+step "s2_begin" { BEGIN; }
+step "s2_drop_schema" { DROP SCHEMA testschema; }
+step "s2_drop_foo_type" { DROP TYPE public.foo; }
+step "s2_drop_footab_type" { DROP TYPE public.footab; }
+step "s2_commit" { COMMIT; }
+
+permutation "s1_begin" "s1_create_function_in_schema" "s2_drop_schema" "s1_commit"
+permutation "s2_begin" "s2_drop_schema" "s1_create_function_in_schema" "s2_commit"
+permutation "s1_begin" "s1_create_function_with_type" "s2_drop_foo_type" "s1_commit"
+permutation "s2_begin" "s2_drop_foo_type" "s1_create_function_with_type" "s2_commit"
+permutation "s1_begin" "s1_create_table_with_type" "s2_drop_footab_type" "s1_commit"
+permutation "s2_begin" "s2_drop_footab_type" "s1_create_table_with_type" "s2_commit"
-- 
2.34.1



^ permalink  raw  reply  [nested|flat] 9+ messages in thread

* Re: Avoid orphaned objects dependencies, take 3
@ 2024-04-24 08:38  Bertrand Drouvot <[email protected]>
  parent: Bertrand Drouvot <[email protected]>
  0 siblings, 1 reply; 9+ messages in thread

From: Bertrand Drouvot @ 2024-04-24 08:38 UTC (permalink / raw)
  To: Alexander Lakhin <[email protected]>; +Cc: [email protected]

Hi,

On Tue, Apr 23, 2024 at 04:20:46PM +0000, Bertrand Drouvot wrote:
> Please find attached v2 that should not produce the issue anymore (I launched a
> lot of attempts without any issues). v1 was not strong enough as it was not
> always checking for the dependent object existence. v2 now always checks if the
> object still exists after the additional lock acquisition attempt while recording
> the dependency.
> 
> I still need to think about v2 but in the meantime could you please also give
> v2 a try on you side?

I gave more thought to v2 and the approach seems reasonable to me. Basically what
it does is that in case the object is already dropped before we take the new lock
(while recording the dependency) then the error message is a "generic" one (means
it does not provide the description of the "already" dropped object). I think it
makes sense to write the patch that way by abandoning the patch's ambition to
tell the description of the dropped object in all the cases.

Of course, I would be happy to hear others thought about it.

Please find v3 attached (which is v2 with more comments).

Regards,

-- 
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com


Attachments:

  [text/x-diff] v3-0001-Avoid-orphaned-objects-dependencies.patch (14.5K, ../../[email protected]/2-v3-0001-Avoid-orphaned-objects-dependencies.patch)
  download | inline diff:
From 32945912ceddad6b171fd8b345adefa4432af357 Mon Sep 17 00:00:00 2001
From: Bertrand Drouvot <[email protected]>
Date: Fri, 29 Mar 2024 15:43:26 +0000
Subject: [PATCH v3] Avoid orphaned objects dependencies

It's currently possible to create orphaned objects dependencies, for example:

Scenario 1:

session 1: begin; drop schema schem;
session 2: create a function in the schema schem
session 1: commit;

With the above, the function created in session 2 would be linked to a non
existing schema.

Scenario 2:

session 1: begin; create a function in the schema schem
session 2: drop schema schem;
session 1: commit;

With the above, the function created in session 1 would be linked to a non
existing schema.

To avoid those scenarios, a new lock (that conflicts with a lock taken by DROP)
has been put in place when the dependencies are being recorded. With this in
place, the drop schema in scenario 2 would be locked.

Also, after locking the object, the patch checks that the object still exists:
with this in place session 2 in scenario 1 would be locked and would report an
error once session 1 committs (that would not be the case should session 1 abort
the transaction).

The patch adds a few tests for some dependency cases (that would currently produce
orphaned objects):

- schema and function (as the above scenarios)
- function and type
- table and type
---
 src/backend/catalog/dependency.c              | 54 ++++++++++++++
 src/backend/catalog/objectaddress.c           | 70 +++++++++++++++++++
 src/backend/catalog/pg_depend.c               |  6 ++
 src/include/catalog/dependency.h              |  1 +
 src/include/catalog/objectaddress.h           |  1 +
 src/test/modules/Makefile                     |  1 +
 src/test/modules/meson.build                  |  1 +
 .../test_dependencies_locks/.gitignore        |  3 +
 .../modules/test_dependencies_locks/Makefile  | 14 ++++
 .../expected/test_dependencies_locks.out      | 49 +++++++++++++
 .../test_dependencies_locks/meson.build       | 12 ++++
 .../specs/test_dependencies_locks.spec        | 39 +++++++++++
 12 files changed, 251 insertions(+)
  40.5% src/backend/catalog/
  29.6% src/test/modules/test_dependencies_locks/expected/
  18.7% src/test/modules/test_dependencies_locks/specs/
   8.3% src/test/modules/test_dependencies_locks/

diff --git a/src/backend/catalog/dependency.c b/src/backend/catalog/dependency.c
index d4b5b2ade1..a49357bbe2 100644
--- a/src/backend/catalog/dependency.c
+++ b/src/backend/catalog/dependency.c
@@ -1517,6 +1517,60 @@ AcquireDeletionLock(const ObjectAddress *object, int flags)
 	}
 }
 
+/*
+ * depLockAndCheckObject
+ *
+ * Lock the object that we are about to record a dependency on.
+ * After it's locked, verify that it hasn't been dropped while we
+ * weren't looking.  If the object has been dropped, this function
+ * does not return!
+ */
+void
+depLockAndCheckObject(const ObjectAddress *object)
+{
+	char	   *object_description;
+
+	/*
+	 * Those don't rely on LockDatabaseObject() when being dropped (see
+	 * AcquireDeletionLock()). Also it looks like they can not produce
+	 * orphaned dependent objects when being dropped.
+	 */
+	if (object->classId == RelationRelationId || object->classId == AuthMemRelationId)
+		return;
+
+	object_description = getObjectDescription(object, true);
+
+	/* assume we should lock the whole object not a sub-object */
+	LockDatabaseObject(object->classId, object->objectId, 0, AccessShareLock);
+
+	/* check if object still exists */
+	if (!ObjectByIdExist(object, false))
+	{
+		/*
+		 * It might be possible that we are creating it (for example creating
+		 * a composite type while creating a relation), so bypass the syscache
+		 * lookup and use a dirty snaphot instead to cover this scenario.
+		 */
+		if (!ObjectByIdExist(object, true))
+		{
+			/*
+			 * If the object has been dropped before we get a chance to get
+			 * its description, then emit a generic error message. That looks
+			 * like a good compromise over extra complexity.
+			 */
+			if (object_description)
+				ereport(ERROR, errmsg("%s does not exist", object_description));
+			else
+				ereport(ERROR, errmsg("a dependent object does not exist"));
+		}
+	}
+
+	if (object_description)
+		pfree(object_description);
+
+	return;
+}
+
 /*
  * ReleaseDeletionLock - release an object deletion lock
  *
diff --git a/src/backend/catalog/objectaddress.c b/src/backend/catalog/objectaddress.c
index 7b536ac6fd..c2b873dd81 100644
--- a/src/backend/catalog/objectaddress.c
+++ b/src/backend/catalog/objectaddress.c
@@ -2590,6 +2590,76 @@ get_object_namespace(const ObjectAddress *address)
 	return oid;
 }
 
+/*
+ * ObjectByIdExist
+ *
+ * Return whether the given object exists.
+ *
+ * Works for most catalogs, if no special processing is needed.
+ */
+bool
+ObjectByIdExist(const ObjectAddress *address, bool use_dirty_snapshot)
+{
+	HeapTuple	tuple;
+	int			cache = -1;
+	const ObjectPropertyType *property;
+
+	if (!use_dirty_snapshot)
+	{
+		property = get_object_property_data(address->classId);
+
+		cache = property->oid_catcache_id;
+	}
+
+	if (cache >= 0)
+	{
+		/* Fetch tuple from syscache. */
+		tuple = SearchSysCache1(cache, ObjectIdGetDatum(address->objectId));
+
+		if (!HeapTupleIsValid(tuple))
+		{
+			return false;
+		}
+
+		ReleaseSysCache(tuple);
+
+		return true;
+	}
+	else
+	{
+		Relation	rel;
+		ScanKeyData skey[1];
+		SysScanDesc scan;
+		SnapshotData DirtySnapshot;
+		Snapshot	snapshot;
+
+		if (use_dirty_snapshot)
+		{
+			InitDirtySnapshot(DirtySnapshot);
+			snapshot = &DirtySnapshot;
+		}
+		else
+			snapshot = NULL;
+
+		rel = table_open(address->classId, AccessShareLock);
+
+		ScanKeyInit(&skey[0],
+					get_object_attnum_oid(address->classId),
+					BTEqualStrategyNumber, F_OIDEQ,
+					ObjectIdGetDatum(address->objectId));
+
+		scan = systable_beginscan(rel, get_object_oid_index(address->classId), true,
+								  snapshot, 1, skey);
+
+		/* we expect exactly one match */
+		tuple = systable_getnext(scan);
+		systable_endscan(scan);
+		table_close(rel, AccessShareLock);
+
+		return (HeapTupleIsValid(tuple));
+	}
+}
+
 /*
  * Return ObjectType for the given object type as given by
  * getObjectTypeDescription; if no valid ObjectType code exists, but it's a
diff --git a/src/backend/catalog/pg_depend.c b/src/backend/catalog/pg_depend.c
index f85a898de8..6bb218f5dd 100644
--- a/src/backend/catalog/pg_depend.c
+++ b/src/backend/catalog/pg_depend.c
@@ -106,6 +106,12 @@ recordMultipleDependencies(const ObjectAddress *depender,
 		if (isObjectPinned(referenced))
 			continue;
 
+		/*
+		 * Acquire a lock and check object still exists while recording the
+		 * dependency. XXX - Should we do so only for DEPENDENCY_NORMAL?
+		 */
+		depLockAndCheckObject(referenced);
+
 		if (slot_init_count < max_slots)
 		{
 			slot[slot_stored_count] = MakeSingleTupleTableSlot(RelationGetDescr(dependDesc),
diff --git a/src/include/catalog/dependency.h b/src/include/catalog/dependency.h
index ec654010d4..8915548711 100644
--- a/src/include/catalog/dependency.h
+++ b/src/include/catalog/dependency.h
@@ -94,6 +94,7 @@ typedef struct ObjectAddresses ObjectAddresses;
 /* in dependency.c */
 
 extern void AcquireDeletionLock(const ObjectAddress *object, int flags);
+extern void depLockAndCheckObject(const ObjectAddress *object);
 
 extern void ReleaseDeletionLock(const ObjectAddress *object);
 
diff --git a/src/include/catalog/objectaddress.h b/src/include/catalog/objectaddress.h
index 3a70d80e32..04891abcc1 100644
--- a/src/include/catalog/objectaddress.h
+++ b/src/include/catalog/objectaddress.h
@@ -53,6 +53,7 @@ extern void check_object_ownership(Oid roleid,
 								   Node *object, Relation relation);
 
 extern Oid	get_object_namespace(const ObjectAddress *address);
+extern bool ObjectByIdExist(const ObjectAddress *address, bool use_dirty_snapshot);
 
 extern bool is_objectclass_supported(Oid class_id);
 extern const char *get_object_class_descr(Oid class_id);
diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile
index 256799f520..75f357100f 100644
--- a/src/test/modules/Makefile
+++ b/src/test/modules/Makefile
@@ -17,6 +17,7 @@ SUBDIRS = \
 		  test_copy_callbacks \
 		  test_custom_rmgrs \
 		  test_ddl_deparse \
+		  test_dependencies_locks \
 		  test_dsa \
 		  test_dsm_registry \
 		  test_extensions \
diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build
index d8fe059d23..60305dcccd 100644
--- a/src/test/modules/meson.build
+++ b/src/test/modules/meson.build
@@ -16,6 +16,7 @@ subdir('test_bloomfilter')
 subdir('test_copy_callbacks')
 subdir('test_custom_rmgrs')
 subdir('test_ddl_deparse')
+subdir('test_dependencies_locks')
 subdir('test_dsa')
 subdir('test_dsm_registry')
 subdir('test_extensions')
diff --git a/src/test/modules/test_dependencies_locks/.gitignore b/src/test/modules/test_dependencies_locks/.gitignore
new file mode 100644
index 0000000000..bf000faac4
--- /dev/null
+++ b/src/test/modules/test_dependencies_locks/.gitignore
@@ -0,0 +1,3 @@
+# Generated subdirectories
+/log/
+/output_iso
diff --git a/src/test/modules/test_dependencies_locks/Makefile b/src/test/modules/test_dependencies_locks/Makefile
new file mode 100644
index 0000000000..7491048380
--- /dev/null
+++ b/src/test/modules/test_dependencies_locks/Makefile
@@ -0,0 +1,14 @@
+# src/test/modules/test_dependencies_locks/Makefile
+
+ISOLATION = test_dependencies_locks
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = src/test/modules/test_dependencies_locks
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/src/test/modules/test_dependencies_locks/expected/test_dependencies_locks.out b/src/test/modules/test_dependencies_locks/expected/test_dependencies_locks.out
new file mode 100644
index 0000000000..d0980f77d5
--- /dev/null
+++ b/src/test/modules/test_dependencies_locks/expected/test_dependencies_locks.out
@@ -0,0 +1,49 @@
+Parsed test spec with 2 sessions
+
+starting permutation: s1_begin s1_create_function_in_schema s2_drop_schema s1_commit
+step s1_begin: BEGIN;
+step s1_create_function_in_schema: CREATE FUNCTION testschema.foo() RETURNS int AS 'select 1' LANGUAGE sql;
+step s2_drop_schema: DROP SCHEMA testschema; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_schema: <... completed>
+ERROR:  cannot drop schema testschema because other objects depend on it
+
+starting permutation: s2_begin s2_drop_schema s1_create_function_in_schema s2_commit
+step s2_begin: BEGIN;
+step s2_drop_schema: DROP SCHEMA testschema;
+step s1_create_function_in_schema: CREATE FUNCTION testschema.foo() RETURNS int AS 'select 1' LANGUAGE sql; <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_function_in_schema: <... completed>
+ERROR:  schema testschema does not exist
+
+starting permutation: s1_begin s1_create_function_with_type s2_drop_foo_type s1_commit
+step s1_begin: BEGIN;
+step s1_create_function_with_type: CREATE FUNCTION footype(num foo) RETURNS int AS 'select 1' LANGUAGE sql;
+step s2_drop_foo_type: DROP TYPE public.foo; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_foo_type: <... completed>
+ERROR:  cannot drop type foo because other objects depend on it
+
+starting permutation: s2_begin s2_drop_foo_type s1_create_function_with_type s2_commit
+step s2_begin: BEGIN;
+step s2_drop_foo_type: DROP TYPE public.foo;
+step s1_create_function_with_type: CREATE FUNCTION footype(num foo) RETURNS int AS 'select 1' LANGUAGE sql; <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_function_with_type: <... completed>
+ERROR:  type foo does not exist
+
+starting permutation: s1_begin s1_create_table_with_type s2_drop_footab_type s1_commit
+step s1_begin: BEGIN;
+step s1_create_table_with_type: CREATE TABLE tabtype(a footab);
+step s2_drop_footab_type: DROP TYPE public.footab; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_footab_type: <... completed>
+ERROR:  cannot drop type footab because other objects depend on it
+
+starting permutation: s2_begin s2_drop_footab_type s1_create_table_with_type s2_commit
+step s2_begin: BEGIN;
+step s2_drop_footab_type: DROP TYPE public.footab;
+step s1_create_table_with_type: CREATE TABLE tabtype(a footab); <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_table_with_type: <... completed>
+ERROR:  type footab does not exist
diff --git a/src/test/modules/test_dependencies_locks/meson.build b/src/test/modules/test_dependencies_locks/meson.build
new file mode 100644
index 0000000000..92a978ab93
--- /dev/null
+++ b/src/test/modules/test_dependencies_locks/meson.build
@@ -0,0 +1,12 @@
+# Copyright (c) 2024, PostgreSQL Global Development Group
+
+tests += {
+  'name': 'test_dependencies_locks',
+  'sd': meson.current_source_dir(),
+  'bd': meson.current_build_dir(),
+  'isolation': {
+    'specs': [
+      'test_dependencies_locks',
+    ],
+  },
+}
diff --git a/src/test/modules/test_dependencies_locks/specs/test_dependencies_locks.spec b/src/test/modules/test_dependencies_locks/specs/test_dependencies_locks.spec
new file mode 100644
index 0000000000..fd15bd2a78
--- /dev/null
+++ b/src/test/modules/test_dependencies_locks/specs/test_dependencies_locks.spec
@@ -0,0 +1,39 @@
+setup
+{
+  CREATE SCHEMA testschema;
+  CREATE TYPE public.foo as enum ('one', 'two');
+  CREATE TYPE public.footab as enum ('three', 'four');
+}
+
+teardown
+{
+  DROP FUNCTION IF EXISTS testschema.foo();
+  DROP FUNCTION IF EXISTS footype(num foo);
+  DROP TABLE IF EXISTS tabtype;
+  DROP SCHEMA IF EXISTS testschema;
+  DROP TYPE IF EXISTS public.foo;
+  DROP TYPE IF EXISTS public.footab;
+}
+
+session "s1"
+
+step "s1_begin" { BEGIN; }
+step "s1_create_function_in_schema" { CREATE FUNCTION testschema.foo() RETURNS int AS 'select 1' LANGUAGE sql; }
+step "s1_create_function_with_type" { CREATE FUNCTION footype(num foo) RETURNS int AS 'select 1' LANGUAGE sql; }
+step "s1_create_table_with_type" { CREATE TABLE tabtype(a footab); }
+step "s1_commit" { COMMIT; }
+
+session "s2"
+
+step "s2_begin" { BEGIN; }
+step "s2_drop_schema" { DROP SCHEMA testschema; }
+step "s2_drop_foo_type" { DROP TYPE public.foo; }
+step "s2_drop_footab_type" { DROP TYPE public.footab; }
+step "s2_commit" { COMMIT; }
+
+permutation "s1_begin" "s1_create_function_in_schema" "s2_drop_schema" "s1_commit"
+permutation "s2_begin" "s2_drop_schema" "s1_create_function_in_schema" "s2_commit"
+permutation "s1_begin" "s1_create_function_with_type" "s2_drop_foo_type" "s1_commit"
+permutation "s2_begin" "s2_drop_foo_type" "s1_create_function_with_type" "s2_commit"
+permutation "s1_begin" "s1_create_table_with_type" "s2_drop_footab_type" "s1_commit"
+permutation "s2_begin" "s2_drop_footab_type" "s1_create_table_with_type" "s2_commit"
-- 
2.34.1



^ permalink  raw  reply  [nested|flat] 9+ messages in thread

* Re: Avoid orphaned objects dependencies, take 3
@ 2024-04-24 12:00  Alexander Lakhin <[email protected]>
  parent: Bertrand Drouvot <[email protected]>
  0 siblings, 1 reply; 9+ messages in thread

From: Alexander Lakhin @ 2024-04-24 12:00 UTC (permalink / raw)
  To: Bertrand Drouvot <[email protected]>; +Cc: [email protected]

Hi Bertrand,

24.04.2024 11:38, Bertrand Drouvot wrote:
>> Please find attached v2 that should not produce the issue anymore (I launched a
>> lot of attempts without any issues). v1 was not strong enough as it was not
>> always checking for the dependent object existence. v2 now always checks if the
>> object still exists after the additional lock acquisition attempt while recording
>> the dependency.
>>
>> I still need to think about v2 but in the meantime could you please also give
>> v2 a try on you side?
> I gave more thought to v2 and the approach seems reasonable to me. Basically what
> it does is that in case the object is already dropped before we take the new lock
> (while recording the dependency) then the error message is a "generic" one (means
> it does not provide the description of the "already" dropped object). I think it
> makes sense to write the patch that way by abandoning the patch's ambition to
> tell the description of the dropped object in all the cases.
>
> Of course, I would be happy to hear others thought about it.
>
> Please find v3 attached (which is v2 with more comments).

Thank you for the improved version!

I can confirm that it fixes that case.
I've also tested other cases that fail on master (most of them also fail
with v1), please try/look at the attached script. (There could be other
broken-dependency cases, of course, but I think I've covered all the
representative ones.)

All tested cases work correctly with v3 applied — I couldn't get broken
dependencies, though concurrent create/drop operations can still produce
the "cache lookup failed" error, which is probably okay, except that it is
an INTERNAL_ERROR, which assumed to be not easily triggered by users.

Best regards,
Alexander
if [ "$1" == "function-schema" ]; then
res=1
for ((i=1;i<=300;i++)); do
  ( { for ((n=1;n<=20;n++)); do echo "DROP SCHEMA s;"; done } | psql ) >psql1.log 2>&1 & 
  echo "
CREATE SCHEMA s;
CREATE FUNCTION s.func1() RETURNS int LANGUAGE SQL AS 'SELECT 1;';
CREATE FUNCTION s.func2() RETURNS int LANGUAGE SQL AS 'SELECT 2;';
CREATE FUNCTION s.func3() RETURNS int LANGUAGE SQL AS 'SELECT 3;';
CREATE FUNCTION s.func4() RETURNS int LANGUAGE SQL AS 'SELECT 4;';
CREATE FUNCTION s.func5() RETURNS int LANGUAGE SQL AS 'SELECT 5;';
  "  | psql >psql2.log 2>&1 &
  wait
  psql -c "DROP SCHEMA s CASCADE" >psql3.log 2>&1
done
echo "
SELECT pg_identify_object('pg_proc'::regclass, pp.oid, 0), pp.oid FROM pg_proc pp
  LEFT JOIN pg_namespace pn ON pp.pronamespace = pn.oid WHERE pn.oid IS NULL" | psql
sleep 1
grep 'was terminated' server.log || res=0
fi


if [ "$1" == "function-function" ]; then
res=1
for ((i=1;i<=1000;i++)); do
( { for ((n=1;n<=20;n++)); do echo "DROP FUNCTION f();"; done } | psql ) >psql1.log 2>&1 &
( echo "
CREATE FUNCTION f() RETURNS int LANGUAGE SQL RETURN 1;
CREATE FUNCTION f1() RETURNS int LANGUAGE SQL RETURN f() + 1;
CREATE FUNCTION f2() RETURNS int LANGUAGE SQL RETURN f() + 2;
CREATE FUNCTION f3() RETURNS int LANGUAGE SQL RETURN f() + 3;
CREATE FUNCTION f4() RETURNS int LANGUAGE SQL RETURN f() + 4;
CREATE FUNCTION f5() RETURNS int LANGUAGE SQL RETURN f() + 5;
" | psql ) >psql2.log 2>&1 &
wait
echo "ALTER FUNCTION f1() RENAME TO f01; SELECT f01()" | psql 2>&1 | grep 'cache lookup failed' && { echo "on iteration $i"; res=1; break; }

psql -c "DROP FUNCTION f() CASCADE" >psql3.log 2>&1
done

grep -A2 'cache lookup failed' server.log || res=0
fi


if [ "$1" == "function-rettype" ]; then
res=0
for ((i=1;i<=5000;i++)); do
( echo "
CREATE DOMAIN id AS int;
CREATE FUNCTION f1() RETURNS id LANGUAGE SQL RETURN 1;
CREATE FUNCTION f2() RETURNS id LANGUAGE SQL RETURN 2;
CREATE FUNCTION f3() RETURNS id LANGUAGE SQL RETURN 3;
CREATE FUNCTION f4() RETURNS id LANGUAGE SQL RETURN 4;
CREATE FUNCTION f5() RETURNS id LANGUAGE SQL RETURN 5;
" | psql ) >psql1.log 2>&1 &
( echo "SELECT pg_sleep(random() / 100); DROP DOMAIN id"  | psql ) >psql2.log 2>&1 &
wait

psql -c "SELECT f1()" 2>&1 | grep 'cache lookup failed' && { echo "on iteration $i"; res=1; break; }
psql -c "DROP DOMAIN id CASCADE" >psql3.log 2>&1
done

[ $res == 1 ] && psql -c "SELECT pn.oid, proname, pronamespace, proowner, prolang, prorettype FROM pg_proc pp INNER JOIN pg_namespace pn ON (pp.pronamespace = pn.oid) WHERE pn.nspname='public'"
fi


if [ "$1" == "function-argtype" ]; then
res=0
for ((i=1;i<=5000;i++)); do
( echo "
CREATE DOMAIN id AS int;
CREATE FUNCTION f1(i id) RETURNS int LANGUAGE SQL RETURN 1;
CREATE FUNCTION f2(i id) RETURNS int LANGUAGE SQL RETURN 2;
CREATE FUNCTION f3(i id) RETURNS int LANGUAGE SQL RETURN 3;
CREATE FUNCTION f4(i id) RETURNS int LANGUAGE SQL RETURN 4;
CREATE FUNCTION f5(i id) RETURNS int LANGUAGE SQL RETURN 5;
" | psql ) >psql1.log 2>&1 &
( echo "SELECT pg_sleep(random() / 1000); DROP DOMAIN id" | psql ) >psql2.log 2>&1 &
wait

psql -c "SELECT f1(1)" 2>&1 | grep 'cache lookup failed' && { echo "on iteration $i"; res=1; break; }
psql -q -c "DROP DOMAIN id CASCADE" >/dev/null 2>&1
done

grep -A1 'cache lookup failed' server.log || res=0
fi


if [ "$1" == "domain-domain" ]; then
res=0
for ((i=1;i<=30;i++)); do
{ echo "CREATE DOMAIN id AS int;"; for ((d=1;d<100;d++)); do echo "CREATE DOMAIN id$d AS id;"; done; echo "DROP DOMAIN id CASCADE;"; } | psql >psql0.log 2>&1
{ for ((n=1;n<=100;n++)); do echo "CREATE DOMAIN id AS int;"; for ((d=1;d<100;d++)); do echo "CREATE DOMAIN id$d AS id;"; done; echo "DROP DOMAIN id CASCADE;"; done; } | ON_ERROR_STOP=1 psql >psql1.log 2>&1 &
{ for ((n=1;n<=100;n++)); do echo "SELECT pg_sleep(random() / 100); DROP DOMAIN id;"; done; } | psql >psql2.log 2>&1 &
wait
  
psql -c "CREATE TABLE t1(i id1)" 2>&1 | grep 'cache lookup failed' && { echo "on iteration $i"; res=1; break; }
done
grep -A1 'cache lookup failed' server.log
psql -c "\\dD+ i*"
fi


if [ "$1" == "table-trigger" ]; then
res=1
for ((i=1;i<=100;i++)); do
psql -c "DROP TABLE IF EXISTS t"
psql -c "CREATE TABLE t(i int, c text)"
( { for ((n=1;n<=30;n++)); do echo "DROP FUNCTION trigger_func();"; done } | psql ) >>psql1.log 2>&1 &
( echo "
CREATE FUNCTION trigger_func() RETURNS trigger LANGUAGE plpgsql AS '
BEGIN
    RAISE NOTICE ''trigger_func(%) called: action = %, when = %, level = %'', TG_ARGV[0], TG_OP, TG_WHEN, TG_LEVEL;
    RETURN NULL;
END;';
CREATE TRIGGER modified_c1 BEFORE UPDATE OF c ON t
FOR EACH ROW WHEN (OLD.c <> NEW.c) EXECUTE PROCEDURE trigger_func('modified_c');
CREATE TRIGGER modified_c2 BEFORE UPDATE OF c ON t
FOR EACH ROW WHEN (OLD.c <> NEW.c) EXECUTE PROCEDURE trigger_func('modified_c');
CREATE TRIGGER modified_c3 BEFORE UPDATE OF c ON t
FOR EACH ROW WHEN (OLD.c <> NEW.c) EXECUTE PROCEDURE trigger_func('modified_c');
CREATE TRIGGER modified_c4 BEFORE UPDATE OF c ON t
FOR EACH ROW WHEN (OLD.c <> NEW.c) EXECUTE PROCEDURE trigger_func('modified_c');
CREATE TRIGGER modified_c5 BEFORE UPDATE OF c ON t
FOR EACH ROW WHEN (OLD.c <> NEW.c) EXECUTE PROCEDURE trigger_func('modified_c');
" | psql ) >>psql2.log 2>&1 &
wait
psql -c "DROP FUNCTION trigger_func() CASCADE" >psql3.log 2>&1

psql -c "INSERT INTO t VALUES(1, 'a')"
psql -c "UPDATE t SET c='b'" 2>&1 | grep 'cache lookup failed' && { echo "on iteration $i"; res=1; break; }
done

grep -A2 'cache lookup failed' server.log || res=0
fi


if [ "$1" == "table-coltype" ]; then
res=0
numclients=20
for ((i=1;i<=50;i++)); do
for ((c=1;c<=numclients;c++)); do
  echo "SELECT pg_sleep(random() / 2000); DROP DOMAIN id_$c RESTRICT" | psql >psql1-$c.log 2>&1 &
  echo "
CREATE DOMAIN id_$c int;
CREATE TABLE tbl_$c (i id_$c);
  " | psql >psql2-$c.log 2>&1 &
done
wait
for ((c=1;c<=numclients;c++)); do
  echo "SELECT * FROM tbl_$c" | psql 2>&1 | grep 'cache lookup failed' && { echo "on iteration $i"; res=1; break; }
  echo "DROP DOMAIN id_$c CASCADE; DROP TABLE tbl_$c" | psql >psql3-$c.log 2>&1
done
[ $res == 1 ] && break;
done
grep -A2 'cache lookup failed' server.log
fi


if [ "$1" == "type-shelltype" ]; then
res=0
numclients=40
for ((i=1;i<=100;i++)); do
for ((c=1;c<=numclients;c++)); do
  echo "DROP FUNCTION IF EXISTS xint8in_$c, xint8out_$c CASCADE" | psql >psql0-$c.log 2>&1
done
for ((c=1;c<=numclients;c++)); do
  echo "
CREATE TYPE xint8_$c;
CREATE FUNCTION xint8in_$c(cstring) RETURNS xint8_$c IMMUTABLE STRICT LANGUAGE INTERNAL AS 'int8in';
CREATE FUNCTION xint8out_$c(xint8_$c) RETURNS cstring IMMUTABLE STRICT LANGUAGE INTERNAL AS 'int8out';
CREATE TYPE xint8_$c(input = xint8in_$c, output = xint8out_$c, like = int8);
  " | psql >psql1-$c.log 2>&1 &
  echo "SELECT pg_sleep(random() / 1000); DROP TYPE xint8_$c CASCADE;" | psql >psql2-$c.log 2>&1 &
done
wait
grep 'TRAP' server.log && { echo "on iteration $i"; res=1; break; }
done
fi


if [ "$1" == "type-schema" ]; then
res=0
numclients=20
for ((i=1;i<=100;i++)); do
for ((c=1;c<=numclients;c++)); do
  echo "
CREATE SCHEMA s_$c;
CREATE DOMAIN s_$c.dom_$c int4;
CREATE TABLE tbl_$c (i dom_$c);
  " | psql >psql1-$c.log 2>&1 &
  echo "DROP SCHEMA s_$c RESTRICT" | psql >psql2-$c.log 2>&1 &
done
wait
pg_dump -f db.dump || { echo "on iteration $i"; res=1; break; }
for ((c=1;c<=numclients;c++)); do
  echo "DROP SCHEMA s_$c CASCADE" | psql >psql3-$c.log 2>&1
done
done
fi


if [ "$1" == "ts_template-schema" ]; then
res=0
numclients=20
for ((i=1;i<=100;i++)); do
for ((c=1;c<=numclients;c++)); do
  echo "
CREATE SCHEMA s_$c;
CREATE TEXT SEARCH TEMPLATE s_$c.tst_$c(lexize=dsimple_lexize);
  " | psql >psql1-$c.log 2>&1 &
  echo "DROP SCHEMA s_$c RESTRICT;" | psql >psql2-$c.log 2>&1 &
done
wait
pg_dump -f db.dump || { echo "on iteration $i"; res=1; break; }
for ((c=1;c<=numclients;c++)); do
  echo "DROP SCHEMA s_$c CASCADE;" | psql >psql3-$c.log 2>&1
done
done
fi


if [ "$1" == "cast-schema" ]; then
res=0
numclients=20
for ((i=1;i<=100;i++)); do
for ((c=1;c<=numclients;c++)); do
  echo "
CREATE SCHEMA s_$c;
CREATE DOMAIN s_$c.dom_$c int4;
CREATE TABLE tbl_$c (i dom_$c);
  " | psql >psql1-$c.log 2>&1 &
  echo "DROP SCHEMA s_$c RESTRICT;" | psql >psql2-$c.log 2>&1 &
done
wait
pg_dump -f db.dump || { echo "on iteration $i"; res=1; break; }
for ((c=1;c<=numclients;c++)); do
  echo "DROP SCHEMA s_$c CASCADE;" | psql >psql3-$c.log 2>&1
done
done
fi


if [ "$1" == "cast-function" ]; then
res=0
echo "
CREATE TYPE tt;
CREATE FUNCTION tt_in(cstring) RETURNS tt AS 'textin' LANGUAGE internal STRICT IMMUTABLE;
CREATE FUNCTION tt_out(tt) RETURNS cstring AS 'textout' LANGUAGE internal STRICT IMMUTABLE;

CREATE TYPE tt (internallength = variable, input = tt_in, output = tt_out, alignment = int4);
" | psql

numclients=2
for ((n=1;n<=500;n++)); do
  for ((i=1;i<=$numclients;i++)); do
cat << 'EOF' | psql >psql-$i.log 2>&1 &
DROP CAST (int4 AS tt);
CREATE FUNCTION cf(int4) RETURNS tt LANGUAGE SQL AS
  $$ SELECT ($1::text)::tt; $$;
CREATE CAST (int4 AS tt) WITH FUNCTION cf(int4) AS IMPLICIT;
DROP FUNCTION cf(int4) CASCADE;
EOF
  done
  wait
  echo "SELECT 1234::int4::tt;" | psql 2>&1 | grep 'cache lookup failed for function'
  pg_dump -f db.dump || { echo "on iteration $i"; res=1; break; }
done
fi


if [ "$1" == "server-fdw_wrapper" ]; then
res=0
numclients=20
for ((i=1;i<=200;i++)); do
for ((c=1;c<=numclients;c++)); do
  echo "
CREATE FOREIGN DATA WRAPPER fdw_wrapper_$c;
CREATE SERVER tst_$c FOREIGN DATA WRAPPER fdw_wrapper_$c;
  " | psql >psql1-$c.log 2>&1 &
  echo "DROP FOREIGN DATA WRAPPER fdw_wrapper_$c RESTRICT;" | psql >psql2-$c.log 2>&1 &
done
wait
pg_dump -f db.dump || { echo "on iteration $i"; res=1; break; }
for ((c=1;c<=numclients;c++)); do
  echo "DROP FOREIGN DATA WRAPPER fdw_wrapper_$c CASCADE;" | psql >psql3-$c.log 2>&1
done
done
fi


Attachments:

  [text/plain] test-broken-deps.sh.txt (9.5K, ../../[email protected]/2-test-broken-deps.sh.txt)
  download | inline:
if [ "$1" == "function-schema" ]; then
res=1
for ((i=1;i<=300;i++)); do
  ( { for ((n=1;n<=20;n++)); do echo "DROP SCHEMA s;"; done } | psql ) >psql1.log 2>&1 & 
  echo "
CREATE SCHEMA s;
CREATE FUNCTION s.func1() RETURNS int LANGUAGE SQL AS 'SELECT 1;';
CREATE FUNCTION s.func2() RETURNS int LANGUAGE SQL AS 'SELECT 2;';
CREATE FUNCTION s.func3() RETURNS int LANGUAGE SQL AS 'SELECT 3;';
CREATE FUNCTION s.func4() RETURNS int LANGUAGE SQL AS 'SELECT 4;';
CREATE FUNCTION s.func5() RETURNS int LANGUAGE SQL AS 'SELECT 5;';
  "  | psql >psql2.log 2>&1 &
  wait
  psql -c "DROP SCHEMA s CASCADE" >psql3.log 2>&1
done
echo "
SELECT pg_identify_object('pg_proc'::regclass, pp.oid, 0), pp.oid FROM pg_proc pp
  LEFT JOIN pg_namespace pn ON pp.pronamespace = pn.oid WHERE pn.oid IS NULL" | psql
sleep 1
grep 'was terminated' server.log || res=0
fi


if [ "$1" == "function-function" ]; then
res=1
for ((i=1;i<=1000;i++)); do
( { for ((n=1;n<=20;n++)); do echo "DROP FUNCTION f();"; done } | psql ) >psql1.log 2>&1 &
( echo "
CREATE FUNCTION f() RETURNS int LANGUAGE SQL RETURN 1;
CREATE FUNCTION f1() RETURNS int LANGUAGE SQL RETURN f() + 1;
CREATE FUNCTION f2() RETURNS int LANGUAGE SQL RETURN f() + 2;
CREATE FUNCTION f3() RETURNS int LANGUAGE SQL RETURN f() + 3;
CREATE FUNCTION f4() RETURNS int LANGUAGE SQL RETURN f() + 4;
CREATE FUNCTION f5() RETURNS int LANGUAGE SQL RETURN f() + 5;
" | psql ) >psql2.log 2>&1 &
wait
echo "ALTER FUNCTION f1() RENAME TO f01; SELECT f01()" | psql 2>&1 | grep 'cache lookup failed' && { echo "on iteration $i"; res=1; break; }

psql -c "DROP FUNCTION f() CASCADE" >psql3.log 2>&1
done

grep -A2 'cache lookup failed' server.log || res=0
fi


if [ "$1" == "function-rettype" ]; then
res=0
for ((i=1;i<=5000;i++)); do
( echo "
CREATE DOMAIN id AS int;
CREATE FUNCTION f1() RETURNS id LANGUAGE SQL RETURN 1;
CREATE FUNCTION f2() RETURNS id LANGUAGE SQL RETURN 2;
CREATE FUNCTION f3() RETURNS id LANGUAGE SQL RETURN 3;
CREATE FUNCTION f4() RETURNS id LANGUAGE SQL RETURN 4;
CREATE FUNCTION f5() RETURNS id LANGUAGE SQL RETURN 5;
" | psql ) >psql1.log 2>&1 &
( echo "SELECT pg_sleep(random() / 100); DROP DOMAIN id"  | psql ) >psql2.log 2>&1 &
wait

psql -c "SELECT f1()" 2>&1 | grep 'cache lookup failed' && { echo "on iteration $i"; res=1; break; }
psql -c "DROP DOMAIN id CASCADE" >psql3.log 2>&1
done

[ $res == 1 ] && psql -c "SELECT pn.oid, proname, pronamespace, proowner, prolang, prorettype FROM pg_proc pp INNER JOIN pg_namespace pn ON (pp.pronamespace = pn.oid) WHERE pn.nspname='public'"
fi


if [ "$1" == "function-argtype" ]; then
res=0
for ((i=1;i<=5000;i++)); do
( echo "
CREATE DOMAIN id AS int;
CREATE FUNCTION f1(i id) RETURNS int LANGUAGE SQL RETURN 1;
CREATE FUNCTION f2(i id) RETURNS int LANGUAGE SQL RETURN 2;
CREATE FUNCTION f3(i id) RETURNS int LANGUAGE SQL RETURN 3;
CREATE FUNCTION f4(i id) RETURNS int LANGUAGE SQL RETURN 4;
CREATE FUNCTION f5(i id) RETURNS int LANGUAGE SQL RETURN 5;
" | psql ) >psql1.log 2>&1 &
( echo "SELECT pg_sleep(random() / 1000); DROP DOMAIN id" | psql ) >psql2.log 2>&1 &
wait

psql -c "SELECT f1(1)" 2>&1 | grep 'cache lookup failed' && { echo "on iteration $i"; res=1; break; }
psql -q -c "DROP DOMAIN id CASCADE" >/dev/null 2>&1
done

grep -A1 'cache lookup failed' server.log || res=0
fi


if [ "$1" == "domain-domain" ]; then
res=0
for ((i=1;i<=30;i++)); do
{ echo "CREATE DOMAIN id AS int;"; for ((d=1;d<100;d++)); do echo "CREATE DOMAIN id$d AS id;"; done; echo "DROP DOMAIN id CASCADE;"; } | psql >psql0.log 2>&1
{ for ((n=1;n<=100;n++)); do echo "CREATE DOMAIN id AS int;"; for ((d=1;d<100;d++)); do echo "CREATE DOMAIN id$d AS id;"; done; echo "DROP DOMAIN id CASCADE;"; done; } | ON_ERROR_STOP=1 psql >psql1.log 2>&1 &
{ for ((n=1;n<=100;n++)); do echo "SELECT pg_sleep(random() / 100); DROP DOMAIN id;"; done; } | psql >psql2.log 2>&1 &
wait
  
psql -c "CREATE TABLE t1(i id1)" 2>&1 | grep 'cache lookup failed' && { echo "on iteration $i"; res=1; break; }
done
grep -A1 'cache lookup failed' server.log
psql -c "\\dD+ i*"
fi


if [ "$1" == "table-trigger" ]; then
res=1
for ((i=1;i<=100;i++)); do
psql -c "DROP TABLE IF EXISTS t"
psql -c "CREATE TABLE t(i int, c text)"
( { for ((n=1;n<=30;n++)); do echo "DROP FUNCTION trigger_func();"; done } | psql ) >>psql1.log 2>&1 &
( echo "
CREATE FUNCTION trigger_func() RETURNS trigger LANGUAGE plpgsql AS '
BEGIN
    RAISE NOTICE ''trigger_func(%) called: action = %, when = %, level = %'', TG_ARGV[0], TG_OP, TG_WHEN, TG_LEVEL;
    RETURN NULL;
END;';
CREATE TRIGGER modified_c1 BEFORE UPDATE OF c ON t
FOR EACH ROW WHEN (OLD.c <> NEW.c) EXECUTE PROCEDURE trigger_func('modified_c');
CREATE TRIGGER modified_c2 BEFORE UPDATE OF c ON t
FOR EACH ROW WHEN (OLD.c <> NEW.c) EXECUTE PROCEDURE trigger_func('modified_c');
CREATE TRIGGER modified_c3 BEFORE UPDATE OF c ON t
FOR EACH ROW WHEN (OLD.c <> NEW.c) EXECUTE PROCEDURE trigger_func('modified_c');
CREATE TRIGGER modified_c4 BEFORE UPDATE OF c ON t
FOR EACH ROW WHEN (OLD.c <> NEW.c) EXECUTE PROCEDURE trigger_func('modified_c');
CREATE TRIGGER modified_c5 BEFORE UPDATE OF c ON t
FOR EACH ROW WHEN (OLD.c <> NEW.c) EXECUTE PROCEDURE trigger_func('modified_c');
" | psql ) >>psql2.log 2>&1 &
wait
psql -c "DROP FUNCTION trigger_func() CASCADE" >psql3.log 2>&1

psql -c "INSERT INTO t VALUES(1, 'a')"
psql -c "UPDATE t SET c='b'" 2>&1 | grep 'cache lookup failed' && { echo "on iteration $i"; res=1; break; }
done

grep -A2 'cache lookup failed' server.log || res=0
fi


if [ "$1" == "table-coltype" ]; then
res=0
numclients=20
for ((i=1;i<=50;i++)); do
for ((c=1;c<=numclients;c++)); do
  echo "SELECT pg_sleep(random() / 2000); DROP DOMAIN id_$c RESTRICT" | psql >psql1-$c.log 2>&1 &
  echo "
CREATE DOMAIN id_$c int;
CREATE TABLE tbl_$c (i id_$c);
  " | psql >psql2-$c.log 2>&1 &
done
wait
for ((c=1;c<=numclients;c++)); do
  echo "SELECT * FROM tbl_$c" | psql 2>&1 | grep 'cache lookup failed' && { echo "on iteration $i"; res=1; break; }
  echo "DROP DOMAIN id_$c CASCADE; DROP TABLE tbl_$c" | psql >psql3-$c.log 2>&1
done
[ $res == 1 ] && break;
done
grep -A2 'cache lookup failed' server.log
fi


if [ "$1" == "type-shelltype" ]; then
res=0
numclients=40
for ((i=1;i<=100;i++)); do
for ((c=1;c<=numclients;c++)); do
  echo "DROP FUNCTION IF EXISTS xint8in_$c, xint8out_$c CASCADE" | psql >psql0-$c.log 2>&1
done
for ((c=1;c<=numclients;c++)); do
  echo "
CREATE TYPE xint8_$c;
CREATE FUNCTION xint8in_$c(cstring) RETURNS xint8_$c IMMUTABLE STRICT LANGUAGE INTERNAL AS 'int8in';
CREATE FUNCTION xint8out_$c(xint8_$c) RETURNS cstring IMMUTABLE STRICT LANGUAGE INTERNAL AS 'int8out';
CREATE TYPE xint8_$c(input = xint8in_$c, output = xint8out_$c, like = int8);
  " | psql >psql1-$c.log 2>&1 &
  echo "SELECT pg_sleep(random() / 1000); DROP TYPE xint8_$c CASCADE;" | psql >psql2-$c.log 2>&1 &
done
wait
grep 'TRAP' server.log && { echo "on iteration $i"; res=1; break; }
done
fi


if [ "$1" == "type-schema" ]; then
res=0
numclients=20
for ((i=1;i<=100;i++)); do
for ((c=1;c<=numclients;c++)); do
  echo "
CREATE SCHEMA s_$c;
CREATE DOMAIN s_$c.dom_$c int4;
CREATE TABLE tbl_$c (i dom_$c);
  " | psql >psql1-$c.log 2>&1 &
  echo "DROP SCHEMA s_$c RESTRICT" | psql >psql2-$c.log 2>&1 &
done
wait
pg_dump -f db.dump || { echo "on iteration $i"; res=1; break; }
for ((c=1;c<=numclients;c++)); do
  echo "DROP SCHEMA s_$c CASCADE" | psql >psql3-$c.log 2>&1
done
done
fi


if [ "$1" == "ts_template-schema" ]; then
res=0
numclients=20
for ((i=1;i<=100;i++)); do
for ((c=1;c<=numclients;c++)); do
  echo "
CREATE SCHEMA s_$c;
CREATE TEXT SEARCH TEMPLATE s_$c.tst_$c(lexize=dsimple_lexize);
  " | psql >psql1-$c.log 2>&1 &
  echo "DROP SCHEMA s_$c RESTRICT;" | psql >psql2-$c.log 2>&1 &
done
wait
pg_dump -f db.dump || { echo "on iteration $i"; res=1; break; }
for ((c=1;c<=numclients;c++)); do
  echo "DROP SCHEMA s_$c CASCADE;" | psql >psql3-$c.log 2>&1
done
done
fi


if [ "$1" == "cast-schema" ]; then
res=0
numclients=20
for ((i=1;i<=100;i++)); do
for ((c=1;c<=numclients;c++)); do
  echo "
CREATE SCHEMA s_$c;
CREATE DOMAIN s_$c.dom_$c int4;
CREATE TABLE tbl_$c (i dom_$c);
  " | psql >psql1-$c.log 2>&1 &
  echo "DROP SCHEMA s_$c RESTRICT;" | psql >psql2-$c.log 2>&1 &
done
wait
pg_dump -f db.dump || { echo "on iteration $i"; res=1; break; }
for ((c=1;c<=numclients;c++)); do
  echo "DROP SCHEMA s_$c CASCADE;" | psql >psql3-$c.log 2>&1
done
done
fi


if [ "$1" == "cast-function" ]; then
res=0
echo "
CREATE TYPE tt;
CREATE FUNCTION tt_in(cstring) RETURNS tt AS 'textin' LANGUAGE internal STRICT IMMUTABLE;
CREATE FUNCTION tt_out(tt) RETURNS cstring AS 'textout' LANGUAGE internal STRICT IMMUTABLE;

CREATE TYPE tt (internallength = variable, input = tt_in, output = tt_out, alignment = int4);
" | psql

numclients=2
for ((n=1;n<=500;n++)); do
  for ((i=1;i<=$numclients;i++)); do
cat << 'EOF' | psql >psql-$i.log 2>&1 &
DROP CAST (int4 AS tt);
CREATE FUNCTION cf(int4) RETURNS tt LANGUAGE SQL AS
  $$ SELECT ($1::text)::tt; $$;
CREATE CAST (int4 AS tt) WITH FUNCTION cf(int4) AS IMPLICIT;
DROP FUNCTION cf(int4) CASCADE;
EOF
  done
  wait
  echo "SELECT 1234::int4::tt;" | psql 2>&1 | grep 'cache lookup failed for function'
  pg_dump -f db.dump || { echo "on iteration $i"; res=1; break; }
done
fi


if [ "$1" == "server-fdw_wrapper" ]; then
res=0
numclients=20
for ((i=1;i<=200;i++)); do
for ((c=1;c<=numclients;c++)); do
  echo "
CREATE FOREIGN DATA WRAPPER fdw_wrapper_$c;
CREATE SERVER tst_$c FOREIGN DATA WRAPPER fdw_wrapper_$c;
  " | psql >psql1-$c.log 2>&1 &
  echo "DROP FOREIGN DATA WRAPPER fdw_wrapper_$c RESTRICT;" | psql >psql2-$c.log 2>&1 &
done
wait
pg_dump -f db.dump || { echo "on iteration $i"; res=1; break; }
for ((c=1;c<=numclients;c++)); do
  echo "DROP FOREIGN DATA WRAPPER fdw_wrapper_$c CASCADE;" | psql >psql3-$c.log 2>&1
done
done
fi

^ permalink  raw  reply  [nested|flat] 9+ messages in thread

* Re: Avoid orphaned objects dependencies, take 3
@ 2024-04-25 05:00  Bertrand Drouvot <[email protected]>
  parent: Alexander Lakhin <[email protected]>
  0 siblings, 1 reply; 9+ messages in thread

From: Bertrand Drouvot @ 2024-04-25 05:00 UTC (permalink / raw)
  To: Alexander Lakhin <[email protected]>; +Cc: [email protected]

Hi,

On Wed, Apr 24, 2024 at 03:00:00PM +0300, Alexander Lakhin wrote:
> 24.04.2024 11:38, Bertrand Drouvot wrote:
> > I gave more thought to v2 and the approach seems reasonable to me. Basically what
> > it does is that in case the object is already dropped before we take the new lock
> > (while recording the dependency) then the error message is a "generic" one (means
> > it does not provide the description of the "already" dropped object). I think it
> > makes sense to write the patch that way by abandoning the patch's ambition to
> > tell the description of the dropped object in all the cases.
> > 
> > Of course, I would be happy to hear others thought about it.
> > 
> > Please find v3 attached (which is v2 with more comments).
> 
> Thank you for the improved version!
> 
> I can confirm that it fixes that case.

Great, thanks for the test!

> I've also tested other cases that fail on master (most of them also fail
> with v1), please try/look at the attached script.

Thanks for all those tests!

> (There could be other broken-dependency cases, of course, but I think I've
> covered all the representative ones.)

Agree. Furthermore the way the patch is written should be agnostic to the
object's kind that are part of the dependency. Having said that, that does not
hurt to add more tests in this patch, so v4 attached adds some of your tests (
that would fail on the master branch without the patch applied).

The way the tests are written in the patch are less "racy" that when triggered
with your script. Indeed, I think that in the patch the dependent object can not
be removed before the new lock is taken when recording the dependency. We may
want to add injection points in the game if we feel the need.

> All tested cases work correctly with v3 applied —

Yeah, same on my side, I did run them too and did not find any issues.

> I couldn't get broken
> dependencies,

Same here.

> though concurrent create/drop operations can still produce
> the "cache lookup failed" error, which is probably okay, except that it is
> an INTERNAL_ERROR, which assumed to be not easily triggered by users.

I did not see any of those "cache lookup failed" during my testing with/without
your script. During which test(s) did you see those with v3 applied?

Attached v4, simply adding more tests to v3.

Regards,

-- 
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com


Attachments:

  [text/x-diff] v4-0001-Avoid-orphaned-objects-dependencies.patch (19.8K, ../../[email protected]/2-v4-0001-Avoid-orphaned-objects-dependencies.patch)
  download | inline diff:
From a9b34955fab0351b7b5a7ba6eb36f199f5a5822c Mon Sep 17 00:00:00 2001
From: Bertrand Drouvot <[email protected]>
Date: Fri, 29 Mar 2024 15:43:26 +0000
Subject: [PATCH v4] Avoid orphaned objects dependencies

It's currently possible to create orphaned objects dependencies, for example:

Scenario 1:

session 1: begin; drop schema schem;
session 2: create a function in the schema schem
session 1: commit;

With the above, the function created in session 2 would be linked to a non
existing schema.

Scenario 2:

session 1: begin; create a function in the schema schem
session 2: drop schema schem;
session 1: commit;

With the above, the function created in session 1 would be linked to a non
existing schema.

To avoid those scenarios, a new lock (that conflicts with a lock taken by DROP)
has been put in place when the dependencies are being recorded. With this in
place, the drop schema in scenario 2 would be locked.

Also, after the new lock attempt, the patch checks that the object still exists:
with this in place session 2 in scenario 1 would be locked and would report an
error once session 1 committs (that would not be the case should session 1 abort
the transaction).

If the object is dropped before the new lock attempt is triggered then the patch
would also report an error (but with less details).

The patch adds a few tests for some dependency cases (that would currently produce
orphaned objects):

- schema and function (as the above scenarios)
- function and arg type
- function and return type
- function and function
- domain and domain
- table and type
- server and foreign data wrapper
---
 src/backend/catalog/dependency.c              |  54 +++++++++
 src/backend/catalog/objectaddress.c           |  70 +++++++++++
 src/backend/catalog/pg_depend.c               |   6 +
 src/include/catalog/dependency.h              |   1 +
 src/include/catalog/objectaddress.h           |   1 +
 src/test/modules/Makefile                     |   1 +
 src/test/modules/meson.build                  |   1 +
 .../test_dependencies_locks/.gitignore        |   3 +
 .../modules/test_dependencies_locks/Makefile  |  14 +++
 .../expected/test_dependencies_locks.out      | 113 ++++++++++++++++++
 .../test_dependencies_locks/meson.build       |  12 ++
 .../specs/test_dependencies_locks.spec        |  78 ++++++++++++
 12 files changed, 354 insertions(+)
  24.6% src/backend/catalog/
  42.7% src/test/modules/test_dependencies_locks/expected/
  25.9% src/test/modules/test_dependencies_locks/specs/
   5.0% src/test/modules/test_dependencies_locks/

diff --git a/src/backend/catalog/dependency.c b/src/backend/catalog/dependency.c
index d4b5b2ade1..a49357bbe2 100644
--- a/src/backend/catalog/dependency.c
+++ b/src/backend/catalog/dependency.c
@@ -1517,6 +1517,60 @@ AcquireDeletionLock(const ObjectAddress *object, int flags)
 	}
 }
 
+/*
+ * depLockAndCheckObject
+ *
+ * Lock the object that we are about to record a dependency on.
+ * After it's locked, verify that it hasn't been dropped while we
+ * weren't looking.  If the object has been dropped, this function
+ * does not return!
+ */
+void
+depLockAndCheckObject(const ObjectAddress *object)
+{
+	char	   *object_description;
+
+	/*
+	 * Those don't rely on LockDatabaseObject() when being dropped (see
+	 * AcquireDeletionLock()). Also it looks like they can not produce
+	 * orphaned dependent objects when being dropped.
+	 */
+	if (object->classId == RelationRelationId || object->classId == AuthMemRelationId)
+		return;
+
+	object_description = getObjectDescription(object, true);
+
+	/* assume we should lock the whole object not a sub-object */
+	LockDatabaseObject(object->classId, object->objectId, 0, AccessShareLock);
+
+	/* check if object still exists */
+	if (!ObjectByIdExist(object, false))
+	{
+		/*
+		 * It might be possible that we are creating it (for example creating
+		 * a composite type while creating a relation), so bypass the syscache
+		 * lookup and use a dirty snaphot instead to cover this scenario.
+		 */
+		if (!ObjectByIdExist(object, true))
+		{
+			/*
+			 * If the object has been dropped before we get a chance to get
+			 * its description, then emit a generic error message. That looks
+			 * like a good compromise over extra complexity.
+			 */
+			if (object_description)
+				ereport(ERROR, errmsg("%s does not exist", object_description));
+			else
+				ereport(ERROR, errmsg("a dependent object does not exist"));
+		}
+	}
+
+	if (object_description)
+		pfree(object_description);
+
+	return;
+}
+
 /*
  * ReleaseDeletionLock - release an object deletion lock
  *
diff --git a/src/backend/catalog/objectaddress.c b/src/backend/catalog/objectaddress.c
index 7b536ac6fd..c2b873dd81 100644
--- a/src/backend/catalog/objectaddress.c
+++ b/src/backend/catalog/objectaddress.c
@@ -2590,6 +2590,76 @@ get_object_namespace(const ObjectAddress *address)
 	return oid;
 }
 
+/*
+ * ObjectByIdExist
+ *
+ * Return whether the given object exists.
+ *
+ * Works for most catalogs, if no special processing is needed.
+ */
+bool
+ObjectByIdExist(const ObjectAddress *address, bool use_dirty_snapshot)
+{
+	HeapTuple	tuple;
+	int			cache = -1;
+	const ObjectPropertyType *property;
+
+	if (!use_dirty_snapshot)
+	{
+		property = get_object_property_data(address->classId);
+
+		cache = property->oid_catcache_id;
+	}
+
+	if (cache >= 0)
+	{
+		/* Fetch tuple from syscache. */
+		tuple = SearchSysCache1(cache, ObjectIdGetDatum(address->objectId));
+
+		if (!HeapTupleIsValid(tuple))
+		{
+			return false;
+		}
+
+		ReleaseSysCache(tuple);
+
+		return true;
+	}
+	else
+	{
+		Relation	rel;
+		ScanKeyData skey[1];
+		SysScanDesc scan;
+		SnapshotData DirtySnapshot;
+		Snapshot	snapshot;
+
+		if (use_dirty_snapshot)
+		{
+			InitDirtySnapshot(DirtySnapshot);
+			snapshot = &DirtySnapshot;
+		}
+		else
+			snapshot = NULL;
+
+		rel = table_open(address->classId, AccessShareLock);
+
+		ScanKeyInit(&skey[0],
+					get_object_attnum_oid(address->classId),
+					BTEqualStrategyNumber, F_OIDEQ,
+					ObjectIdGetDatum(address->objectId));
+
+		scan = systable_beginscan(rel, get_object_oid_index(address->classId), true,
+								  snapshot, 1, skey);
+
+		/* we expect exactly one match */
+		tuple = systable_getnext(scan);
+		systable_endscan(scan);
+		table_close(rel, AccessShareLock);
+
+		return (HeapTupleIsValid(tuple));
+	}
+}
+
 /*
  * Return ObjectType for the given object type as given by
  * getObjectTypeDescription; if no valid ObjectType code exists, but it's a
diff --git a/src/backend/catalog/pg_depend.c b/src/backend/catalog/pg_depend.c
index f85a898de8..6bb218f5dd 100644
--- a/src/backend/catalog/pg_depend.c
+++ b/src/backend/catalog/pg_depend.c
@@ -106,6 +106,12 @@ recordMultipleDependencies(const ObjectAddress *depender,
 		if (isObjectPinned(referenced))
 			continue;
 
+		/*
+		 * Acquire a lock and check object still exists while recording the
+		 * dependency. XXX - Should we do so only for DEPENDENCY_NORMAL?
+		 */
+		depLockAndCheckObject(referenced);
+
 		if (slot_init_count < max_slots)
 		{
 			slot[slot_stored_count] = MakeSingleTupleTableSlot(RelationGetDescr(dependDesc),
diff --git a/src/include/catalog/dependency.h b/src/include/catalog/dependency.h
index ec654010d4..8915548711 100644
--- a/src/include/catalog/dependency.h
+++ b/src/include/catalog/dependency.h
@@ -94,6 +94,7 @@ typedef struct ObjectAddresses ObjectAddresses;
 /* in dependency.c */
 
 extern void AcquireDeletionLock(const ObjectAddress *object, int flags);
+extern void depLockAndCheckObject(const ObjectAddress *object);
 
 extern void ReleaseDeletionLock(const ObjectAddress *object);
 
diff --git a/src/include/catalog/objectaddress.h b/src/include/catalog/objectaddress.h
index 3a70d80e32..04891abcc1 100644
--- a/src/include/catalog/objectaddress.h
+++ b/src/include/catalog/objectaddress.h
@@ -53,6 +53,7 @@ extern void check_object_ownership(Oid roleid,
 								   Node *object, Relation relation);
 
 extern Oid	get_object_namespace(const ObjectAddress *address);
+extern bool ObjectByIdExist(const ObjectAddress *address, bool use_dirty_snapshot);
 
 extern bool is_objectclass_supported(Oid class_id);
 extern const char *get_object_class_descr(Oid class_id);
diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile
index 256799f520..75f357100f 100644
--- a/src/test/modules/Makefile
+++ b/src/test/modules/Makefile
@@ -17,6 +17,7 @@ SUBDIRS = \
 		  test_copy_callbacks \
 		  test_custom_rmgrs \
 		  test_ddl_deparse \
+		  test_dependencies_locks \
 		  test_dsa \
 		  test_dsm_registry \
 		  test_extensions \
diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build
index d8fe059d23..60305dcccd 100644
--- a/src/test/modules/meson.build
+++ b/src/test/modules/meson.build
@@ -16,6 +16,7 @@ subdir('test_bloomfilter')
 subdir('test_copy_callbacks')
 subdir('test_custom_rmgrs')
 subdir('test_ddl_deparse')
+subdir('test_dependencies_locks')
 subdir('test_dsa')
 subdir('test_dsm_registry')
 subdir('test_extensions')
diff --git a/src/test/modules/test_dependencies_locks/.gitignore b/src/test/modules/test_dependencies_locks/.gitignore
new file mode 100644
index 0000000000..bf000faac4
--- /dev/null
+++ b/src/test/modules/test_dependencies_locks/.gitignore
@@ -0,0 +1,3 @@
+# Generated subdirectories
+/log/
+/output_iso
diff --git a/src/test/modules/test_dependencies_locks/Makefile b/src/test/modules/test_dependencies_locks/Makefile
new file mode 100644
index 0000000000..7491048380
--- /dev/null
+++ b/src/test/modules/test_dependencies_locks/Makefile
@@ -0,0 +1,14 @@
+# src/test/modules/test_dependencies_locks/Makefile
+
+ISOLATION = test_dependencies_locks
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = src/test/modules/test_dependencies_locks
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/src/test/modules/test_dependencies_locks/expected/test_dependencies_locks.out b/src/test/modules/test_dependencies_locks/expected/test_dependencies_locks.out
new file mode 100644
index 0000000000..93318c9cbb
--- /dev/null
+++ b/src/test/modules/test_dependencies_locks/expected/test_dependencies_locks.out
@@ -0,0 +1,113 @@
+Parsed test spec with 2 sessions
+
+starting permutation: s1_begin s1_create_function_in_schema s2_drop_schema s1_commit
+step s1_begin: BEGIN;
+step s1_create_function_in_schema: CREATE FUNCTION testschema.foo() RETURNS int AS 'select 1' LANGUAGE sql;
+step s2_drop_schema: DROP SCHEMA testschema; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_schema: <... completed>
+ERROR:  cannot drop schema testschema because other objects depend on it
+
+starting permutation: s2_begin s2_drop_schema s1_create_function_in_schema s2_commit
+step s2_begin: BEGIN;
+step s2_drop_schema: DROP SCHEMA testschema;
+step s1_create_function_in_schema: CREATE FUNCTION testschema.foo() RETURNS int AS 'select 1' LANGUAGE sql; <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_function_in_schema: <... completed>
+ERROR:  schema testschema does not exist
+
+starting permutation: s1_begin s1_create_function_with_argtype s2_drop_foo_type s1_commit
+step s1_begin: BEGIN;
+step s1_create_function_with_argtype: CREATE FUNCTION fooargtype(num foo) RETURNS int AS 'select 1' LANGUAGE sql;
+step s2_drop_foo_type: DROP TYPE public.foo; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_foo_type: <... completed>
+ERROR:  cannot drop type foo because other objects depend on it
+
+starting permutation: s2_begin s2_drop_foo_type s1_create_function_with_argtype s2_commit
+step s2_begin: BEGIN;
+step s2_drop_foo_type: DROP TYPE public.foo;
+step s1_create_function_with_argtype: CREATE FUNCTION fooargtype(num foo) RETURNS int AS 'select 1' LANGUAGE sql; <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_function_with_argtype: <... completed>
+ERROR:  type foo does not exist
+
+starting permutation: s1_begin s1_create_function_with_rettype s2_drop_foo_rettype s1_commit
+step s1_begin: BEGIN;
+step s1_create_function_with_rettype: CREATE FUNCTION footrettype() RETURNS id LANGUAGE sql RETURN 1;
+step s2_drop_foo_rettype: DROP DOMAIN id; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_foo_rettype: <... completed>
+ERROR:  cannot drop type id because other objects depend on it
+
+starting permutation: s2_begin s2_drop_foo_rettype s1_create_function_with_rettype s2_commit
+step s2_begin: BEGIN;
+step s2_drop_foo_rettype: DROP DOMAIN id;
+step s1_create_function_with_rettype: CREATE FUNCTION footrettype() RETURNS id LANGUAGE sql RETURN 1; <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_function_with_rettype: <... completed>
+ERROR:  type id does not exist
+
+starting permutation: s1_begin s1_create_function_with_function s2_drop_function_f s1_commit
+step s1_begin: BEGIN;
+step s1_create_function_with_function: CREATE FUNCTION foofunc() RETURNS int LANGUAGE SQL RETURN f() + 1;
+step s2_drop_function_f: DROP FUNCTION f(); <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_function_f: <... completed>
+ERROR:  cannot drop function f() because other objects depend on it
+
+starting permutation: s2_begin s2_drop_function_f s1_create_function_with_function s2_commit
+step s2_begin: BEGIN;
+step s2_drop_function_f: DROP FUNCTION f();
+step s1_create_function_with_function: CREATE FUNCTION foofunc() RETURNS int LANGUAGE SQL RETURN f() + 1; <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_function_with_function: <... completed>
+ERROR:  function f() does not exist
+
+starting permutation: s1_begin s1_create_domain_with_domain s2_drop_domain_id s1_commit
+step s1_begin: BEGIN;
+step s1_create_domain_with_domain: CREATE DOMAIN idid as id;
+step s2_drop_domain_id: DROP DOMAIN id; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_domain_id: <... completed>
+ERROR:  cannot drop type id because other objects depend on it
+
+starting permutation: s2_begin s2_drop_domain_id s1_create_domain_with_domain s2_commit
+step s2_begin: BEGIN;
+step s2_drop_domain_id: DROP DOMAIN id;
+step s1_create_domain_with_domain: CREATE DOMAIN idid as id; <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_domain_with_domain: <... completed>
+ERROR:  type id does not exist
+
+starting permutation: s1_begin s1_create_table_with_type s2_drop_footab_type s1_commit
+step s1_begin: BEGIN;
+step s1_create_table_with_type: CREATE TABLE tabtype(a footab);
+step s2_drop_footab_type: DROP TYPE public.footab; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_footab_type: <... completed>
+ERROR:  cannot drop type footab because other objects depend on it
+
+starting permutation: s2_begin s2_drop_footab_type s1_create_table_with_type s2_commit
+step s2_begin: BEGIN;
+step s2_drop_footab_type: DROP TYPE public.footab;
+step s1_create_table_with_type: CREATE TABLE tabtype(a footab); <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_table_with_type: <... completed>
+ERROR:  type footab does not exist
+
+starting permutation: s1_begin s1_create_server_with_fdw_wrapper s2_drop_fdw_wrapper s1_commit
+step s1_begin: BEGIN;
+step s1_create_server_with_fdw_wrapper: CREATE SERVER srv_fdw_wrapper FOREIGN DATA WRAPPER fdw_wrapper;
+step s2_drop_fdw_wrapper: DROP FOREIGN DATA WRAPPER fdw_wrapper RESTRICT; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_fdw_wrapper: <... completed>
+ERROR:  cannot drop foreign-data wrapper fdw_wrapper because other objects depend on it
+
+starting permutation: s2_begin s2_drop_fdw_wrapper s1_create_server_with_fdw_wrapper s2_commit
+step s2_begin: BEGIN;
+step s2_drop_fdw_wrapper: DROP FOREIGN DATA WRAPPER fdw_wrapper RESTRICT;
+step s1_create_server_with_fdw_wrapper: CREATE SERVER srv_fdw_wrapper FOREIGN DATA WRAPPER fdw_wrapper; <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_server_with_fdw_wrapper: <... completed>
+ERROR:  foreign-data wrapper fdw_wrapper does not exist
diff --git a/src/test/modules/test_dependencies_locks/meson.build b/src/test/modules/test_dependencies_locks/meson.build
new file mode 100644
index 0000000000..92a978ab93
--- /dev/null
+++ b/src/test/modules/test_dependencies_locks/meson.build
@@ -0,0 +1,12 @@
+# Copyright (c) 2024, PostgreSQL Global Development Group
+
+tests += {
+  'name': 'test_dependencies_locks',
+  'sd': meson.current_source_dir(),
+  'bd': meson.current_build_dir(),
+  'isolation': {
+    'specs': [
+      'test_dependencies_locks',
+    ],
+  },
+}
diff --git a/src/test/modules/test_dependencies_locks/specs/test_dependencies_locks.spec b/src/test/modules/test_dependencies_locks/specs/test_dependencies_locks.spec
new file mode 100644
index 0000000000..3d90a29c47
--- /dev/null
+++ b/src/test/modules/test_dependencies_locks/specs/test_dependencies_locks.spec
@@ -0,0 +1,78 @@
+setup
+{
+  CREATE SCHEMA testschema;
+  CREATE TYPE public.foo as enum ('one', 'two');
+  CREATE TYPE public.footab as enum ('three', 'four');
+  CREATE DOMAIN id AS int;
+  CREATE FUNCTION f() RETURNS int LANGUAGE SQL RETURN 1;
+  CREATE FOREIGN DATA WRAPPER fdw_wrapper;
+}
+
+teardown
+{
+  DROP FUNCTION IF EXISTS testschema.foo();
+  DROP FUNCTION IF EXISTS fooargtype(num foo);
+  DROP FUNCTION IF EXISTS footrettype();
+  DROP FUNCTION IF EXISTS foofunc();
+  DROP DOMAIN IF EXISTS idid;
+  DROP SERVER IF EXISTS srv_fdw_wrapper;
+  DROP TABLE IF EXISTS tabtype;
+  DROP SCHEMA IF EXISTS testschema;
+  DROP TYPE IF EXISTS public.foo;
+  DROP TYPE IF EXISTS public.footab;
+  DROP DOMAIN IF EXISTS id;
+  DROP FUNCTION IF EXISTS f();
+  DROP FOREIGN DATA WRAPPER IF EXISTS fdw_wrapper;
+}
+
+session "s1"
+
+step "s1_begin" { BEGIN; }
+step "s1_create_function_in_schema" { CREATE FUNCTION testschema.foo() RETURNS int AS 'select 1' LANGUAGE sql; }
+step "s1_create_function_with_argtype" { CREATE FUNCTION fooargtype(num foo) RETURNS int AS 'select 1' LANGUAGE sql; }
+step "s1_create_function_with_rettype" { CREATE FUNCTION footrettype() RETURNS id LANGUAGE sql RETURN 1; }
+step "s1_create_function_with_function" { CREATE FUNCTION foofunc() RETURNS int LANGUAGE SQL RETURN f() + 1; }
+step "s1_create_domain_with_domain" { CREATE DOMAIN idid as id; }
+step "s1_create_table_with_type" { CREATE TABLE tabtype(a footab); }
+step "s1_create_server_with_fdw_wrapper" { CREATE SERVER srv_fdw_wrapper FOREIGN DATA WRAPPER fdw_wrapper; }
+step "s1_commit" { COMMIT; }
+
+session "s2"
+
+step "s2_begin" { BEGIN; }
+step "s2_drop_schema" { DROP SCHEMA testschema; }
+step "s2_drop_foo_type" { DROP TYPE public.foo; }
+step "s2_drop_foo_rettype" { DROP DOMAIN id; }
+step "s2_drop_footab_type" { DROP TYPE public.footab; }
+step "s2_drop_function_f" { DROP FUNCTION f(); }
+step "s2_drop_domain_id" { DROP DOMAIN id; }
+step "s2_drop_fdw_wrapper" { DROP FOREIGN DATA WRAPPER fdw_wrapper RESTRICT; }
+step "s2_commit" { COMMIT; }
+
+# function - schema
+permutation "s1_begin" "s1_create_function_in_schema" "s2_drop_schema" "s1_commit"
+permutation "s2_begin" "s2_drop_schema" "s1_create_function_in_schema" "s2_commit"
+
+# function - argtype
+permutation "s1_begin" "s1_create_function_with_argtype" "s2_drop_foo_type" "s1_commit"
+permutation "s2_begin" "s2_drop_foo_type" "s1_create_function_with_argtype" "s2_commit"
+
+# function - rettype
+permutation "s1_begin" "s1_create_function_with_rettype" "s2_drop_foo_rettype" "s1_commit"
+permutation "s2_begin" "s2_drop_foo_rettype" "s1_create_function_with_rettype" "s2_commit"
+
+# function - function
+permutation "s1_begin" "s1_create_function_with_function" "s2_drop_function_f" "s1_commit"
+permutation "s2_begin" "s2_drop_function_f" "s1_create_function_with_function" "s2_commit"
+
+# domain - domain
+permutation "s1_begin" "s1_create_domain_with_domain" "s2_drop_domain_id" "s1_commit"
+permutation "s2_begin" "s2_drop_domain_id" "s1_create_domain_with_domain" "s2_commit"
+
+# table - type
+permutation "s1_begin" "s1_create_table_with_type" "s2_drop_footab_type" "s1_commit"
+permutation "s2_begin" "s2_drop_footab_type" "s1_create_table_with_type" "s2_commit"
+
+# server - foreign data wrapper
+permutation "s1_begin" "s1_create_server_with_fdw_wrapper" "s2_drop_fdw_wrapper" "s1_commit"
+permutation "s2_begin" "s2_drop_fdw_wrapper" "s1_create_server_with_fdw_wrapper" "s2_commit"
-- 
2.34.1



^ permalink  raw  reply  [nested|flat] 9+ messages in thread

* Re: Avoid orphaned objects dependencies, take 3
@ 2024-04-25 06:00  Alexander Lakhin <[email protected]>
  parent: Bertrand Drouvot <[email protected]>
  0 siblings, 0 replies; 9+ messages in thread

From: Alexander Lakhin @ 2024-04-25 06:00 UTC (permalink / raw)
  To: Bertrand Drouvot <[email protected]>; +Cc: [email protected]

Hi,

25.04.2024 08:00, Bertrand Drouvot wrote:
>
>> though concurrent create/drop operations can still produce
>> the "cache lookup failed" error, which is probably okay, except that it is
>> an INTERNAL_ERROR, which assumed to be not easily triggered by users.
> I did not see any of those "cache lookup failed" during my testing with/without
> your script. During which test(s) did you see those with v3 applied?

You can try, for example, table-trigger, or other tests that check for
"cache lookup failed" psql output only (maybe you'll need to increase the
iteration count). For instance, I've got (with v4 applied):
2024-04-25 05:48:08.102 UTC [3638763] ERROR:  cache lookup failed for function 20893
2024-04-25 05:48:08.102 UTC [3638763] STATEMENT:  CREATE TRIGGER modified_c1 BEFORE UPDATE OF c ON t
         FOR EACH ROW WHEN (OLD.c <> NEW.c) EXECUTE PROCEDURE trigger_func('modified_c');

Or with function-function:
2024-04-25 05:52:31.390 UTC [3711897] ERROR:  cache lookup failed for function 32190 at character 54
2024-04-25 05:52:31.390 UTC [3711897] STATEMENT:  CREATE FUNCTION f1() RETURNS int LANGUAGE SQL RETURN f() + 1;
--
2024-04-25 05:52:37.639 UTC [3720011] ERROR:  cache lookup failed for function 34465 at character 54
2024-04-25 05:52:37.639 UTC [3720011] STATEMENT:  CREATE FUNCTION f1() RETURNS int LANGUAGE SQL RETURN f() + 1;

> Attached v4, simply adding more tests to v3.

Thank you for working on this!

Best regards,
Alexander

^ permalink  raw  reply  [nested|flat] 9+ messages in thread

end of thread, other threads:[~2024-04-25 06:00 UTC | newest]

Thread overview: 9+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2020-10-07 03:11 [PATCH v8 4/8] Propagate changes to indisclustered to child/parents Justin Pryzby <[email protected]>
2024-04-22 10:52 Re: Avoid orphaned objects dependencies, take 3 Bertrand Drouvot <[email protected]>
2024-04-22 12:00 ` Re: Avoid orphaned objects dependencies, take 3 Alexander Lakhin <[email protected]>
2024-04-23 04:59   ` Re: Avoid orphaned objects dependencies, take 3 Bertrand Drouvot <[email protected]>
2024-04-23 16:20     ` Re: Avoid orphaned objects dependencies, take 3 Bertrand Drouvot <[email protected]>
2024-04-24 08:38       ` Re: Avoid orphaned objects dependencies, take 3 Bertrand Drouvot <[email protected]>
2024-04-24 12:00         ` Re: Avoid orphaned objects dependencies, take 3 Alexander Lakhin <[email protected]>
2024-04-25 05:00           ` Re: Avoid orphaned objects dependencies, take 3 Bertrand Drouvot <[email protected]>
2024-04-25 06:00             ` Re: Avoid orphaned objects dependencies, take 3 Alexander Lakhin <[email protected]>

This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox