public inbox for [email protected]
help / color / mirror / Atom feed[PATCH v3 1/1] Adjust assorted hint messages that list all valid options.
13+ messages / 5 participants
[nested] [flat]
* [PATCH v3 1/1] Adjust assorted hint messages that list all valid options.
@ 2022-09-02 21:03 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 13+ messages in thread
From: Nathan Bossart @ 2022-09-02 21:03 UTC (permalink / raw)
Instead of listing all valid options, we now try to provide one
that looks similar. Since this may be useful elsewhere, this
change introduces a new set of functions that can be reused for
similar purposes.
---
contrib/dblink/dblink.c | 26 +++---
contrib/dblink/expected/dblink.out | 1 -
contrib/file_fdw/expected/file_fdw.out | 2 -
contrib/file_fdw/file_fdw.c | 23 ++++--
.../postgres_fdw/expected/postgres_fdw.out | 3 +-
contrib/postgres_fdw/option.c | 22 +++--
src/backend/foreign/foreign.c | 25 ++++--
src/backend/utils/adt/varlena.c | 82 +++++++++++++++++++
src/include/utils/varlena.h | 12 +++
src/test/regress/expected/foreign_data.out | 6 +-
10 files changed, 155 insertions(+), 47 deletions(-)
diff --git a/contrib/dblink/dblink.c b/contrib/dblink/dblink.c
index e323fdd0e6..5e2d7b4c70 100644
--- a/contrib/dblink/dblink.c
+++ b/contrib/dblink/dblink.c
@@ -2008,27 +2008,31 @@ dblink_fdw_validator(PG_FUNCTION_ARGS)
{
/*
* Unknown option, or invalid option for the context specified, so
- * complain about it. Provide a hint with list of valid options
- * for the context.
+ * complain about it. Provide a hint with a valid option that
+ * looks similar, if there is one.
*/
- StringInfoData buf;
const PQconninfoOption *opt;
+ const char *closest_match;
+ ClosestMatchState match_state;
+ bool has_valid_options = false;
- initStringInfo(&buf);
+ initClosestMatch(&match_state, def->defname, 4);
for (opt = options; opt->keyword; opt++)
{
if (is_valid_dblink_option(options, opt->keyword, context))
- appendStringInfo(&buf, "%s%s",
- (buf.len > 0) ? ", " : "",
- opt->keyword);
+ {
+ has_valid_options = true;
+ updateClosestMatch(&match_state, opt->keyword);
+ }
}
+
+ closest_match = getClosestMatch(&match_state);
ereport(ERROR,
(errcode(ERRCODE_FDW_OPTION_NAME_NOT_FOUND),
errmsg("invalid option \"%s\"", def->defname),
- buf.len > 0
- ? errhint("Valid options in this context are: %s",
- buf.data)
- : errhint("There are no valid options in this context.")));
+ has_valid_options ? closest_match ?
+ errhint("Did you mean \"%s\"?", closest_match) : 0 :
+ errhint("There are no valid options in this context.")));
}
}
diff --git a/contrib/dblink/expected/dblink.out b/contrib/dblink/expected/dblink.out
index c7bde6ad07..14d015e4d5 100644
--- a/contrib/dblink/expected/dblink.out
+++ b/contrib/dblink/expected/dblink.out
@@ -897,7 +897,6 @@ $d$;
CREATE USER MAPPING FOR public SERVER fdtest
OPTIONS (server 'localhost'); -- fail, can't specify server here
ERROR: invalid option "server"
-HINT: Valid options in this context are: user, password, sslpassword
CREATE USER MAPPING FOR public SERVER fdtest OPTIONS (user :'USER');
GRANT USAGE ON FOREIGN SERVER fdtest TO regress_dblink_user;
GRANT EXECUTE ON FUNCTION dblink_connect_u(text, text) TO regress_dblink_user;
diff --git a/contrib/file_fdw/expected/file_fdw.out b/contrib/file_fdw/expected/file_fdw.out
index 261af1a8b5..36d76ba26c 100644
--- a/contrib/file_fdw/expected/file_fdw.out
+++ b/contrib/file_fdw/expected/file_fdw.out
@@ -166,7 +166,6 @@ ERROR: invalid option "force_not_null"
HINT: There are no valid options in this context.
CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS (force_not_null '*'); -- ERROR
ERROR: invalid option "force_not_null"
-HINT: Valid options in this context are: filename, program, format, header, delimiter, quote, escape, null, encoding
-- force_null is not allowed to be specified at any foreign object level:
ALTER FOREIGN DATA WRAPPER file_fdw OPTIONS (ADD force_null '*'); -- ERROR
ERROR: invalid option "force_null"
@@ -179,7 +178,6 @@ ERROR: invalid option "force_null"
HINT: There are no valid options in this context.
CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS (force_null '*'); -- ERROR
ERROR: invalid option "force_null"
-HINT: Valid options in this context are: filename, program, format, header, delimiter, quote, escape, null, encoding
-- basic query tests
SELECT * FROM agg_text WHERE b > 10.0 ORDER BY a;
a | b
diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c
index 4773cadec0..53d996bd7c 100644
--- a/contrib/file_fdw/file_fdw.c
+++ b/contrib/file_fdw/file_fdw.c
@@ -37,6 +37,7 @@
#include "utils/memutils.h"
#include "utils/rel.h"
#include "utils/sampling.h"
+#include "utils/varlena.h"
PG_MODULE_MAGIC;
@@ -214,27 +215,31 @@ file_fdw_validator(PG_FUNCTION_ARGS)
if (!is_valid_option(def->defname, catalog))
{
const struct FileFdwOption *opt;
- StringInfoData buf;
+ const char *closest_match;
+ ClosestMatchState match_state;
+ bool has_valid_options = false;
/*
* Unknown option specified, complain about it. Provide a hint
- * with list of valid options for the object.
+ * with a valid option that looks similar, if there is one.
*/
- initStringInfo(&buf);
+ initClosestMatch(&match_state, def->defname, 4);
for (opt = valid_options; opt->optname; opt++)
{
if (catalog == opt->optcontext)
- appendStringInfo(&buf, "%s%s", (buf.len > 0) ? ", " : "",
- opt->optname);
+ {
+ has_valid_options = true;
+ updateClosestMatch(&match_state, opt->optname);
+ }
}
+ closest_match = getClosestMatch(&match_state);
ereport(ERROR,
(errcode(ERRCODE_FDW_INVALID_OPTION_NAME),
errmsg("invalid option \"%s\"", def->defname),
- buf.len > 0
- ? errhint("Valid options in this context are: %s",
- buf.data)
- : errhint("There are no valid options in this context.")));
+ has_valid_options ? closest_match ?
+ errhint("Did you mean \"%s\"?", closest_match) : 0 :
+ errhint("There are no valid options in this context.")));
}
/*
diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out
index 7bf35602b0..2588ad4aa6 100644
--- a/contrib/postgres_fdw/expected/postgres_fdw.out
+++ b/contrib/postgres_fdw/expected/postgres_fdw.out
@@ -188,7 +188,6 @@ ALTER USER MAPPING FOR public SERVER testserver1
ALTER USER MAPPING FOR public SERVER testserver1
OPTIONS (ADD sslmode 'require');
ERROR: invalid option "sslmode"
-HINT: Valid options in this context are: user, password, sslpassword, password_required, sslcert, sslkey
-- But we can add valid ones fine
ALTER USER MAPPING FOR public SERVER testserver1
OPTIONS (ADD sslpassword 'dummy');
@@ -9627,7 +9626,7 @@ DO $d$
END;
$d$;
ERROR: invalid option "password"
-HINT: Valid options in this context are: service, passfile, channel_binding, connect_timeout, dbname, host, hostaddr, port, options, application_name, keepalives, keepalives_idle, keepalives_interval, keepalives_count, tcp_user_timeout, sslmode, sslcompression, sslcert, sslkey, sslrootcert, sslcrl, sslcrldir, sslsni, requirepeer, ssl_min_protocol_version, ssl_max_protocol_version, gssencmode, krbsrvname, gsslib, target_session_attrs, use_remote_estimate, fdw_startup_cost, fdw_tuple_cost, extensions, updatable, truncatable, fetch_size, batch_size, async_capable, parallel_commit, keep_connections
+HINT: Did you mean "passfile"?
CONTEXT: SQL statement "ALTER SERVER loopback_nopw OPTIONS (ADD password 'dummypw')"
PL/pgSQL function inline_code_block line 3 at EXECUTE
-- If we add a password for our user mapping instead, we should get a different
diff --git a/contrib/postgres_fdw/option.c b/contrib/postgres_fdw/option.c
index 95dde056eb..6971c88463 100644
--- a/contrib/postgres_fdw/option.c
+++ b/contrib/postgres_fdw/option.c
@@ -90,26 +90,30 @@ postgres_fdw_validator(PG_FUNCTION_ARGS)
{
/*
* Unknown option specified, complain about it. Provide a hint
- * with list of valid options for the object.
+ * with a valid option that looks similar, if there is one.
*/
PgFdwOption *opt;
- StringInfoData buf;
+ const char *closest_match;
+ ClosestMatchState match_state;
+ bool has_valid_options = false;
- initStringInfo(&buf);
+ initClosestMatch(&match_state, def->defname, 4);
for (opt = postgres_fdw_options; opt->keyword; opt++)
{
if (catalog == opt->optcontext)
- appendStringInfo(&buf, "%s%s", (buf.len > 0) ? ", " : "",
- opt->keyword);
+ {
+ has_valid_options = true;
+ updateClosestMatch(&match_state, opt->keyword);
+ }
}
+ closest_match = getClosestMatch(&match_state);
ereport(ERROR,
(errcode(ERRCODE_FDW_INVALID_OPTION_NAME),
errmsg("invalid option \"%s\"", def->defname),
- buf.len > 0
- ? errhint("Valid options in this context are: %s",
- buf.data)
- : errhint("There are no valid options in this context.")));
+ has_valid_options ? closest_match ?
+ errhint("Did you mean \"%s\"?", closest_match) : 0 :
+ errhint("There are no valid options in this context.")));
}
/*
diff --git a/src/backend/foreign/foreign.c b/src/backend/foreign/foreign.c
index cf222fc3e9..283ebb6a58 100644
--- a/src/backend/foreign/foreign.c
+++ b/src/backend/foreign/foreign.c
@@ -27,6 +27,7 @@
#include "utils/memutils.h"
#include "utils/rel.h"
#include "utils/syscache.h"
+#include "utils/varlena.h"
/*
@@ -621,25 +622,31 @@ postgresql_fdw_validator(PG_FUNCTION_ARGS)
if (!is_conninfo_option(def->defname, catalog))
{
const struct ConnectionOption *opt;
- StringInfoData buf;
+ const char *closest_match;
+ ClosestMatchState match_state;
+ bool has_valid_options = false;
/*
* Unknown option specified, complain about it. Provide a hint
- * with list of valid options for the object.
+ * with a valid option that looks similar, if there is one.
*/
- initStringInfo(&buf);
+ initClosestMatch(&match_state, def->defname, 4);
for (opt = libpq_conninfo_options; opt->optname; opt++)
+ {
if (catalog == opt->optcontext)
- appendStringInfo(&buf, "%s%s", (buf.len > 0) ? ", " : "",
- opt->optname);
+ {
+ has_valid_options = true;
+ updateClosestMatch(&match_state, opt->optname);
+ }
+ }
+ closest_match = getClosestMatch(&match_state);
ereport(ERROR,
(errcode(ERRCODE_SYNTAX_ERROR),
errmsg("invalid option \"%s\"", def->defname),
- buf.len > 0
- ? errhint("Valid options in this context are: %s",
- buf.data)
- : errhint("There are no valid options in this context.")));
+ has_valid_options ? closest_match ?
+ errhint("Did you mean \"%s\"?", closest_match) : 0 :
+ errhint("There are no valid options in this context.")));
PG_RETURN_BOOL(false);
}
diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c
index 8539cef024..174f9c390d 100644
--- a/src/backend/utils/adt/varlena.c
+++ b/src/backend/utils/adt/varlena.c
@@ -6196,6 +6196,88 @@ rest_of_char_same(const char *s1, const char *s2, int len)
#include "levenshtein.c"
+/*
+ * The following *ClosestMatch() functions can be used to determine whether a
+ * user-provided string resembles any known valid values, which is useful for
+ * providing hints in log messages, among other things. Use these functions
+ * like so:
+ *
+ * initClosestMatch(&state, source_string, max_distance);
+ *
+ * for (int i = 0; i < num_valid_strings; i++)
+ * updateClosestMatch(&state, valid_strings[i]);
+ *
+ * closestMatch = getClosestMatch(&state);
+ */
+
+/*
+ * Initialize the given state with the source string and maximum Levenshtein
+ * distance to consider.
+ */
+void
+initClosestMatch(ClosestMatchState *state, const char *source, int max_d)
+{
+ Assert(state);
+ Assert(max_d >= 0);
+
+ state->source = source;
+ state->min_d = -1;
+ state->max_d = max_d;
+ state->match = NULL;
+}
+
+/*
+ * If the candidate string is a closer match than the current one saved (or
+ * there is no match saved), save it as the closest match.
+ *
+ * If the source or candidate string is NULL, empty, or too long, this function
+ * takes no action. Likewise, if the Levenshtein distance exceeds the maximum
+ * allowed or more than half the characters are different, no action is taken.
+ */
+void
+updateClosestMatch(ClosestMatchState *state, const char *candidate)
+{
+ int dist;
+
+ Assert(state);
+
+ if (state->source == NULL || state->source[0] == '\0' ||
+ candidate == NULL || candidate[0] == '\0')
+ return;
+
+ /*
+ * To avoid ERROR-ing, we check the lengths here instead of setting
+ * 'trusted' to false in the call to varstr_levenshtein_less_equal().
+ */
+ if (strlen(state->source) > MAX_LEVENSHTEIN_STRLEN ||
+ strlen(candidate) > MAX_LEVENSHTEIN_STRLEN)
+ return;
+
+ dist = varstr_levenshtein_less_equal(state->source, strlen(state->source),
+ candidate, strlen(candidate), 1, 1, 1,
+ state->max_d, true);
+ if (dist <= state->max_d &&
+ dist <= strlen(state->source) / 2 &&
+ (state->min_d == -1 || dist < state->min_d))
+ {
+ state->min_d = dist;
+ state->match = candidate;
+ }
+}
+
+/*
+ * Return the closest match. If no suitable candidates were provided via
+ * updateClosestMatch(), return NULL.
+ */
+const char *
+getClosestMatch(ClosestMatchState *state)
+{
+ Assert(state);
+
+ return state->match;
+}
+
+
/*
* Unicode support
*/
diff --git a/src/include/utils/varlena.h b/src/include/utils/varlena.h
index c45208a204..2bc3b6e519 100644
--- a/src/include/utils/varlena.h
+++ b/src/include/utils/varlena.h
@@ -38,4 +38,16 @@ extern text *replace_text_regexp(text *src_text, text *pattern_text,
int cflags, Oid collation,
int search_start, int n);
+typedef struct ClosestMatchState
+{
+ const char *source;
+ int min_d;
+ int max_d;
+ const char *match;
+} ClosestMatchState;
+
+extern void initClosestMatch(ClosestMatchState *state, const char *source, int max_d);
+extern void updateClosestMatch(ClosestMatchState *state, const char *candidate);
+extern const char *getClosestMatch(ClosestMatchState *state);
+
#endif
diff --git a/src/test/regress/expected/foreign_data.out b/src/test/regress/expected/foreign_data.out
index 33505352cc..97a0b9cda1 100644
--- a/src/test/regress/expected/foreign_data.out
+++ b/src/test/regress/expected/foreign_data.out
@@ -329,7 +329,6 @@ CREATE SERVER s6 VERSION '16.0' FOREIGN DATA WRAPPER foo OPTIONS (host 'a', dbna
CREATE SERVER s7 TYPE 'oracle' VERSION '17.0' FOREIGN DATA WRAPPER foo OPTIONS (host 'a', dbname 'b');
CREATE SERVER s8 FOREIGN DATA WRAPPER postgresql OPTIONS (foo '1'); -- ERROR
ERROR: invalid option "foo"
-HINT: Valid options in this context are: authtype, service, connect_timeout, dbname, host, hostaddr, port, tty, options, requiressl, sslmode, gsslib
CREATE SERVER s8 FOREIGN DATA WRAPPER postgresql OPTIONS (host 'localhost', dbname 's8db');
\des+
List of foreign servers
@@ -440,7 +439,6 @@ ERROR: permission denied for foreign-data wrapper foo
RESET ROLE;
ALTER SERVER s8 OPTIONS (foo '1'); -- ERROR option validation
ERROR: invalid option "foo"
-HINT: Valid options in this context are: authtype, service, connect_timeout, dbname, host, hostaddr, port, tty, options, requiressl, sslmode, gsslib
ALTER SERVER s8 OPTIONS (connect_timeout '30', SET dbname 'db1', DROP host);
SET ROLE regress_test_role;
ALTER SERVER s1 OWNER TO regress_test_indirect; -- ERROR
@@ -597,7 +595,7 @@ ERROR: user mapping for "regress_foreign_data_user" already exists for server "
CREATE USER MAPPING FOR public SERVER s4 OPTIONS ("this mapping" 'is public');
CREATE USER MAPPING FOR user SERVER s8 OPTIONS (username 'test', password 'secret'); -- ERROR
ERROR: invalid option "username"
-HINT: Valid options in this context are: user, password
+HINT: Did you mean "user"?
CREATE USER MAPPING FOR user SERVER s8 OPTIONS (user 'test', password 'secret');
ALTER SERVER s5 OWNER TO regress_test_role;
ALTER SERVER s6 OWNER TO regress_test_indirect;
@@ -636,7 +634,7 @@ ALTER USER MAPPING FOR public SERVER s5 OPTIONS (gotcha 'true'); -- E
ERROR: user mapping for "public" does not exist for server "s5"
ALTER USER MAPPING FOR current_user SERVER s8 OPTIONS (username 'test'); -- ERROR
ERROR: invalid option "username"
-HINT: Valid options in this context are: user, password
+HINT: Did you mean "user"?
ALTER USER MAPPING FOR current_user SERVER s8 OPTIONS (DROP user, SET password 'public');
SET ROLE regress_test_role;
ALTER USER MAPPING FOR current_user SERVER s5 OPTIONS (ADD modified '1');
--
2.25.1
--gBBFr7Ir9EOA20Yy--
^ permalink raw reply [nested|flat] 13+ messages in thread
* Re: Avoid orphaned objects dependencies, take 3
@ 2024-04-25 07:20 Bertrand Drouvot <[email protected]>
0 siblings, 1 reply; 13+ messages in thread
From: Bertrand Drouvot @ 2024-04-25 07:20 UTC (permalink / raw)
To: Alexander Lakhin <[email protected]>; +Cc: [email protected]
Hi,
On Thu, Apr 25, 2024 at 09:00:00AM +0300, Alexander Lakhin wrote:
> Hi,
>
> 25.04.2024 08:00, Bertrand Drouvot wrote:
> >
> > > though concurrent create/drop operations can still produce
> > > the "cache lookup failed" error, which is probably okay, except that it is
> > > an INTERNAL_ERROR, which assumed to be not easily triggered by users.
> > I did not see any of those "cache lookup failed" during my testing with/without
> > your script. During which test(s) did you see those with v3 applied?
>
> You can try, for example, table-trigger, or other tests that check for
> "cache lookup failed" psql output only (maybe you'll need to increase the
> iteration count). For instance, I've got (with v4 applied):
> 2024-04-25 05:48:08.102 UTC [3638763] ERROR: cache lookup failed for function 20893
> 2024-04-25 05:48:08.102 UTC [3638763] STATEMENT: CREATE TRIGGER modified_c1 BEFORE UPDATE OF c ON t
> FOR EACH ROW WHEN (OLD.c <> NEW.c) EXECUTE PROCEDURE trigger_func('modified_c');
>
> Or with function-function:
> 2024-04-25 05:52:31.390 UTC [3711897] ERROR: cache lookup failed for function 32190 at character 54
> 2024-04-25 05:52:31.390 UTC [3711897] STATEMENT: CREATE FUNCTION f1() RETURNS int LANGUAGE SQL RETURN f() + 1;
> --
> 2024-04-25 05:52:37.639 UTC [3720011] ERROR: cache lookup failed for function 34465 at character 54
> 2024-04-25 05:52:37.639 UTC [3720011] STATEMENT: CREATE FUNCTION f1() RETURNS int LANGUAGE SQL RETURN f() + 1;
I see, so this is during object creation.
It's easy to reproduce this kind of issue with gdb. For example set a breakpoint
on SearchSysCache1() and during the create function f1() once it breaks on:
#0 SearchSysCache1 (cacheId=45, key1=16400) at syscache.c:221
#1 0x00005ad305beacd6 in func_get_detail (funcname=0x5ad308204d50, fargs=0x0, fargnames=0x0, nargs=0, argtypes=0x7ffff2ff9cc0, expand_variadic=true, expand_defaults=true, include_out_arguments=false, funcid=0x7ffff2ff9ba0, rettype=0x7ffff2ff9b9c, retset=0x7ffff2ff9b94, nvargs=0x7ffff2ff9ba4,
vatype=0x7ffff2ff9ba8, true_typeids=0x7ffff2ff9bd8, argdefaults=0x7ffff2ff9be0) at parse_func.c:1622
#2 0x00005ad305be7dd0 in ParseFuncOrColumn (pstate=0x5ad30823be98, funcname=0x5ad308204d50, fargs=0x0, last_srf=0x0, fn=0x5ad308204da0, proc_call=false, location=55) at parse_func.c:266
#3 0x00005ad305bdffb0 in transformFuncCall (pstate=0x5ad30823be98, fn=0x5ad308204da0) at parse_expr.c:1474
#4 0x00005ad305bdd2ee in transformExprRecurse (pstate=0x5ad30823be98, expr=0x5ad308204da0) at parse_expr.c:230
#5 0x00005ad305bdec34 in transformAExprOp (pstate=0x5ad30823be98, a=0x5ad308204e20) at parse_expr.c:990
#6 0x00005ad305bdd1a0 in transformExprRecurse (pstate=0x5ad30823be98, expr=0x5ad308204e20) at parse_expr.c:187
#7 0x00005ad305bdd00b in transformExpr (pstate=0x5ad30823be98, expr=0x5ad308204e20, exprKind=EXPR_KIND_SELECT_TARGET) at parse_expr.c:131
#8 0x00005ad305b96b7e in transformReturnStmt (pstate=0x5ad30823be98, stmt=0x5ad308204ee0) at analyze.c:2395
#9 0x00005ad305b92213 in transformStmt (pstate=0x5ad30823be98, parseTree=0x5ad308204ee0) at analyze.c:375
#10 0x00005ad305c6321a in interpret_AS_clause (languageOid=14, languageName=0x5ad308204c40 "sql", funcname=0x5ad308204ad8 "f100", as=0x0, sql_body_in=0x5ad308204ee0, parameterTypes=0x0, inParameterNames=0x0, prosrc_str_p=0x7ffff2ffa208, probin_str_p=0x7ffff2ffa200, sql_body_out=0x7ffff2ffa210,
queryString=0x5ad3082040b0 "CREATE FUNCTION f100() RETURNS int LANGUAGE SQL RETURN f() + 1;") at functioncmds.c:953
#11 0x00005ad305c63c93 in CreateFunction (pstate=0x5ad308186310, stmt=0x5ad308204f00) at functioncmds.c:1221
then drop function f() in another session. Then the create function f1() would:
postgres=# CREATE FUNCTION f() RETURNS int LANGUAGE SQL RETURN f() + 1;
ERROR: cache lookup failed for function 16400
This stuff does appear before we get a chance to call the new depLockAndCheckObject()
function.
I think this is what Tom was referring to in [1]:
"
So the only real fix for this would be to make every object lookup in the entire
system do the sort of dance that's done in RangeVarGetRelidExtended.
"
The fact that those kind of errors appear also somehow ensure that no orphaned
dependencies can be created.
The patch does ensure that no orphaned depencies can occur after those "initial"
look up are done (should the dependent object be dropped).
I'm tempted to not add extra complexity to avoid those kind of errors and keep the
patch as it is. All of those servicing the same goal: no orphaned depencies are
created.
[1]: https://www.postgresql.org/message-id/2872252.1630851337%40sss.pgh.pa.us
Regards,
--
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 13+ messages in thread
* Re: Avoid orphaned objects dependencies, take 3
@ 2024-04-30 17:00 Alexander Lakhin <[email protected]>
parent: Bertrand Drouvot <[email protected]>
0 siblings, 1 reply; 13+ messages in thread
From: Alexander Lakhin @ 2024-04-30 17:00 UTC (permalink / raw)
To: Bertrand Drouvot <[email protected]>; +Cc: [email protected]
Hi Bertrand,
25.04.2024 10:20, Bertrand Drouvot wrote:
> postgres=# CREATE FUNCTION f() RETURNS int LANGUAGE SQL RETURN f() + 1;
> ERROR: cache lookup failed for function 16400
>
> This stuff does appear before we get a chance to call the new depLockAndCheckObject()
> function.
>
> I think this is what Tom was referring to in [1]:
>
> "
> So the only real fix for this would be to make every object lookup in the entire
> system do the sort of dance that's done in RangeVarGetRelidExtended.
> "
>
> The fact that those kind of errors appear also somehow ensure that no orphaned
> dependencies can be created.
I agree; the only thing that I'd change here, is the error code.
But I've discovered yet another possibility to get a broken dependency.
Please try this script:
res=0
numclients=20
for ((i=1;i<=100;i++)); do
for ((c=1;c<=numclients;c++)); do
echo "
CREATE SCHEMA s_$c;
CREATE CONVERSION myconv_$c FOR 'LATIN1' TO 'UTF8' FROM iso8859_1_to_utf8;
ALTER CONVERSION myconv_$c SET SCHEMA s_$c;
" | psql >psql1-$c.log 2>&1 &
echo "DROP SCHEMA s_$c RESTRICT;" | psql >psql2-$c.log 2>&1 &
done
wait
pg_dump -f db.dump || { echo "on iteration $i"; res=1; break; }
for ((c=1;c<=numclients;c++)); do
echo "DROP SCHEMA s_$c CASCADE;" | psql >psql3-$c.log 2>&1
done
done
psql -c "SELECT * FROM pg_conversion WHERE connamespace NOT IN (SELECT oid FROM pg_namespace);"
It fails for me (with the v4 patch applied) as follows:
pg_dump: error: schema with OID 16392 does not exist
on iteration 1
oid | conname | connamespace | conowner | conforencoding | contoencoding | conproc | condefault
-------+----------+--------------+----------+----------------+---------------+-------------------+------------
16396 | myconv_6 | 16392 | 10 | 8 | 6 | iso8859_1_to_utf8 | f
Best regards,
Alexander
^ permalink raw reply [nested|flat] 13+ messages in thread
* Re: Avoid orphaned objects dependencies, take 3
@ 2024-05-09 12:20 Bertrand Drouvot <[email protected]>
parent: Alexander Lakhin <[email protected]>
0 siblings, 2 replies; 13+ messages in thread
From: Bertrand Drouvot @ 2024-05-09 12:20 UTC (permalink / raw)
To: Alexander Lakhin <[email protected]>; +Cc: [email protected]
Hi,
On Tue, Apr 30, 2024 at 08:00:00PM +0300, Alexander Lakhin wrote:
> Hi Bertrand,
>
> But I've discovered yet another possibility to get a broken dependency.
Thanks for the testing and the report!
> Please try this script:
> res=0
> numclients=20
> for ((i=1;i<=100;i++)); do
> for ((c=1;c<=numclients;c++)); do
> echo "
> CREATE SCHEMA s_$c;
> CREATE CONVERSION myconv_$c FOR 'LATIN1' TO 'UTF8' FROM iso8859_1_to_utf8;
> ALTER CONVERSION myconv_$c SET SCHEMA s_$c;
> " | psql >psql1-$c.log 2>&1 &
> echo "DROP SCHEMA s_$c RESTRICT;" | psql >psql2-$c.log 2>&1 &
> done
> wait
> pg_dump -f db.dump || { echo "on iteration $i"; res=1; break; }
> for ((c=1;c<=numclients;c++)); do
> echo "DROP SCHEMA s_$c CASCADE;" | psql >psql3-$c.log 2>&1
> done
> done
> psql -c "SELECT * FROM pg_conversion WHERE connamespace NOT IN (SELECT oid FROM pg_namespace);"
>
> It fails for me (with the v4 patch applied) as follows:
> pg_dump: error: schema with OID 16392 does not exist
> on iteration 1
> oid | conname | connamespace | conowner | conforencoding | contoencoding | conproc | condefault
> -------+----------+--------------+----------+----------------+---------------+-------------------+------------
> 16396 | myconv_6 | 16392 | 10 | 8 | 6 | iso8859_1_to_utf8 | f
>
Thanks for sharing the test, I'm able to reproduce the issue with v4.
Oh I see, your test updates an existing dependency. v4 took care about brand new
dependencies creation (recordMultipleDependencies()) but forgot to take care
about changing an existing dependency (which is done in another code path:
changeDependencyFor()).
Please find attached v5 that adds:
- a call to the new depLockAndCheckObject() function in changeDependencyFor().
- a test when altering an existing dependency.
With v5 applied, I don't see the issue anymore.
Regards,
--
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
Attachments:
[text/x-diff] v5-0001-Avoid-orphaned-objects-dependencies.patch (21.5K, ../../Zjy%2Fo3t%[email protected]/2-v5-0001-Avoid-orphaned-objects-dependencies.patch)
download | inline diff:
From 6fc24a798210f730ab04833fa58074f142be968e Mon Sep 17 00:00:00 2001
From: Bertrand Drouvot <[email protected]>
Date: Fri, 29 Mar 2024 15:43:26 +0000
Subject: [PATCH v5] Avoid orphaned objects dependencies
It's currently possible to create orphaned objects dependencies, for example:
Scenario 1:
session 1: begin; drop schema schem;
session 2: create a function in the schema schem
session 1: commit;
With the above, the function created in session 2 would be linked to a non
existing schema.
Scenario 2:
session 1: begin; create a function in the schema schem
session 2: drop schema schem;
session 1: commit;
With the above, the function created in session 1 would be linked to a non
existing schema.
To avoid those scenarios, a new lock (that conflicts with a lock taken by DROP)
has been put in place when the dependencies are being recorded. With this in
place, the drop schema in scenario 2 would be locked.
Also, after the new lock attempt, the patch checks that the object still exists:
with this in place session 2 in scenario 1 would be locked and would report an
error once session 1 committs (that would not be the case should session 1 abort
the transaction).
If the object is dropped before the new lock attempt is triggered then the patch
would also report an error (but with less details).
The patch adds a few tests for some dependency cases (that would currently produce
orphaned objects):
- schema and function (as the above scenarios)
- alter a dependency (function and schema)
- function and arg type
- function and return type
- function and function
- domain and domain
- table and type
- server and foreign data wrapper
---
src/backend/catalog/dependency.c | 54 ++++++++
src/backend/catalog/objectaddress.c | 70 ++++++++++
src/backend/catalog/pg_depend.c | 12 ++
src/include/catalog/dependency.h | 1 +
src/include/catalog/objectaddress.h | 1 +
src/test/modules/Makefile | 1 +
src/test/modules/meson.build | 1 +
.../test_dependencies_locks/.gitignore | 3 +
.../modules/test_dependencies_locks/Makefile | 14 ++
.../expected/test_dependencies_locks.out | 129 ++++++++++++++++++
.../test_dependencies_locks/meson.build | 12 ++
.../specs/test_dependencies_locks.spec | 89 ++++++++++++
12 files changed, 387 insertions(+)
22.9% src/backend/catalog/
43.7% src/test/modules/test_dependencies_locks/expected/
27.2% src/test/modules/test_dependencies_locks/specs/
4.5% src/test/modules/test_dependencies_locks/
diff --git a/src/backend/catalog/dependency.c b/src/backend/catalog/dependency.c
index d4b5b2ade1..a49357bbe2 100644
--- a/src/backend/catalog/dependency.c
+++ b/src/backend/catalog/dependency.c
@@ -1517,6 +1517,60 @@ AcquireDeletionLock(const ObjectAddress *object, int flags)
}
}
+/*
+ * depLockAndCheckObject
+ *
+ * Lock the object that we are about to record a dependency on.
+ * After it's locked, verify that it hasn't been dropped while we
+ * weren't looking. If the object has been dropped, this function
+ * does not return!
+ */
+void
+depLockAndCheckObject(const ObjectAddress *object)
+{
+ char *object_description;
+
+ /*
+ * Those don't rely on LockDatabaseObject() when being dropped (see
+ * AcquireDeletionLock()). Also it looks like they can not produce
+ * orphaned dependent objects when being dropped.
+ */
+ if (object->classId == RelationRelationId || object->classId == AuthMemRelationId)
+ return;
+
+ object_description = getObjectDescription(object, true);
+
+ /* assume we should lock the whole object not a sub-object */
+ LockDatabaseObject(object->classId, object->objectId, 0, AccessShareLock);
+
+ /* check if object still exists */
+ if (!ObjectByIdExist(object, false))
+ {
+ /*
+ * It might be possible that we are creating it (for example creating
+ * a composite type while creating a relation), so bypass the syscache
+ * lookup and use a dirty snaphot instead to cover this scenario.
+ */
+ if (!ObjectByIdExist(object, true))
+ {
+ /*
+ * If the object has been dropped before we get a chance to get
+ * its description, then emit a generic error message. That looks
+ * like a good compromise over extra complexity.
+ */
+ if (object_description)
+ ereport(ERROR, errmsg("%s does not exist", object_description));
+ else
+ ereport(ERROR, errmsg("a dependent object does not exist"));
+ }
+ }
+
+ if (object_description)
+ pfree(object_description);
+
+ return;
+}
+
/*
* ReleaseDeletionLock - release an object deletion lock
*
diff --git a/src/backend/catalog/objectaddress.c b/src/backend/catalog/objectaddress.c
index 7b536ac6fd..c2b873dd81 100644
--- a/src/backend/catalog/objectaddress.c
+++ b/src/backend/catalog/objectaddress.c
@@ -2590,6 +2590,76 @@ get_object_namespace(const ObjectAddress *address)
return oid;
}
+/*
+ * ObjectByIdExist
+ *
+ * Return whether the given object exists.
+ *
+ * Works for most catalogs, if no special processing is needed.
+ */
+bool
+ObjectByIdExist(const ObjectAddress *address, bool use_dirty_snapshot)
+{
+ HeapTuple tuple;
+ int cache = -1;
+ const ObjectPropertyType *property;
+
+ if (!use_dirty_snapshot)
+ {
+ property = get_object_property_data(address->classId);
+
+ cache = property->oid_catcache_id;
+ }
+
+ if (cache >= 0)
+ {
+ /* Fetch tuple from syscache. */
+ tuple = SearchSysCache1(cache, ObjectIdGetDatum(address->objectId));
+
+ if (!HeapTupleIsValid(tuple))
+ {
+ return false;
+ }
+
+ ReleaseSysCache(tuple);
+
+ return true;
+ }
+ else
+ {
+ Relation rel;
+ ScanKeyData skey[1];
+ SysScanDesc scan;
+ SnapshotData DirtySnapshot;
+ Snapshot snapshot;
+
+ if (use_dirty_snapshot)
+ {
+ InitDirtySnapshot(DirtySnapshot);
+ snapshot = &DirtySnapshot;
+ }
+ else
+ snapshot = NULL;
+
+ rel = table_open(address->classId, AccessShareLock);
+
+ ScanKeyInit(&skey[0],
+ get_object_attnum_oid(address->classId),
+ BTEqualStrategyNumber, F_OIDEQ,
+ ObjectIdGetDatum(address->objectId));
+
+ scan = systable_beginscan(rel, get_object_oid_index(address->classId), true,
+ snapshot, 1, skey);
+
+ /* we expect exactly one match */
+ tuple = systable_getnext(scan);
+ systable_endscan(scan);
+ table_close(rel, AccessShareLock);
+
+ return (HeapTupleIsValid(tuple));
+ }
+}
+
/*
* Return ObjectType for the given object type as given by
* getObjectTypeDescription; if no valid ObjectType code exists, but it's a
diff --git a/src/backend/catalog/pg_depend.c b/src/backend/catalog/pg_depend.c
index 5366f7820c..f16af28429 100644
--- a/src/backend/catalog/pg_depend.c
+++ b/src/backend/catalog/pg_depend.c
@@ -108,6 +108,12 @@ recordMultipleDependencies(const ObjectAddress *depender,
if (isObjectPinned(referenced))
continue;
+ /*
+ * Acquire a lock and check object still exists while recording the
+ * dependency. XXX - Should we do so only for DEPENDENCY_NORMAL?
+ */
+ depLockAndCheckObject(referenced);
+
if (slot_init_count < max_slots)
{
slot[slot_stored_count] = MakeSingleTupleTableSlot(RelationGetDescr(dependDesc),
@@ -506,6 +512,12 @@ changeDependencyFor(Oid classId, Oid objectId,
return 1;
}
+ /*
+ * Acquire a lock and check object still exists while changing the
+ * dependency.
+ */
+ depLockAndCheckObject(&objAddr);
+
depRel = table_open(DependRelationId, RowExclusiveLock);
/* There should be existing dependency record(s), so search. */
diff --git a/src/include/catalog/dependency.h b/src/include/catalog/dependency.h
index 7eee66f810..5619b6f55e 100644
--- a/src/include/catalog/dependency.h
+++ b/src/include/catalog/dependency.h
@@ -101,6 +101,7 @@ typedef struct ObjectAddresses ObjectAddresses;
/* in dependency.c */
extern void AcquireDeletionLock(const ObjectAddress *object, int flags);
+extern void depLockAndCheckObject(const ObjectAddress *object);
extern void ReleaseDeletionLock(const ObjectAddress *object);
diff --git a/src/include/catalog/objectaddress.h b/src/include/catalog/objectaddress.h
index 3a70d80e32..04891abcc1 100644
--- a/src/include/catalog/objectaddress.h
+++ b/src/include/catalog/objectaddress.h
@@ -53,6 +53,7 @@ extern void check_object_ownership(Oid roleid,
Node *object, Relation relation);
extern Oid get_object_namespace(const ObjectAddress *address);
+extern bool ObjectByIdExist(const ObjectAddress *address, bool use_dirty_snapshot);
extern bool is_objectclass_supported(Oid class_id);
extern const char *get_object_class_descr(Oid class_id);
diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile
index 256799f520..75f357100f 100644
--- a/src/test/modules/Makefile
+++ b/src/test/modules/Makefile
@@ -17,6 +17,7 @@ SUBDIRS = \
test_copy_callbacks \
test_custom_rmgrs \
test_ddl_deparse \
+ test_dependencies_locks \
test_dsa \
test_dsm_registry \
test_extensions \
diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build
index d8fe059d23..60305dcccd 100644
--- a/src/test/modules/meson.build
+++ b/src/test/modules/meson.build
@@ -16,6 +16,7 @@ subdir('test_bloomfilter')
subdir('test_copy_callbacks')
subdir('test_custom_rmgrs')
subdir('test_ddl_deparse')
+subdir('test_dependencies_locks')
subdir('test_dsa')
subdir('test_dsm_registry')
subdir('test_extensions')
diff --git a/src/test/modules/test_dependencies_locks/.gitignore b/src/test/modules/test_dependencies_locks/.gitignore
new file mode 100644
index 0000000000..bf000faac4
--- /dev/null
+++ b/src/test/modules/test_dependencies_locks/.gitignore
@@ -0,0 +1,3 @@
+# Generated subdirectories
+/log/
+/output_iso
diff --git a/src/test/modules/test_dependencies_locks/Makefile b/src/test/modules/test_dependencies_locks/Makefile
new file mode 100644
index 0000000000..7491048380
--- /dev/null
+++ b/src/test/modules/test_dependencies_locks/Makefile
@@ -0,0 +1,14 @@
+# src/test/modules/test_dependencies_locks/Makefile
+
+ISOLATION = test_dependencies_locks
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = src/test/modules/test_dependencies_locks
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/src/test/modules/test_dependencies_locks/expected/test_dependencies_locks.out b/src/test/modules/test_dependencies_locks/expected/test_dependencies_locks.out
new file mode 100644
index 0000000000..9b645d7aa5
--- /dev/null
+++ b/src/test/modules/test_dependencies_locks/expected/test_dependencies_locks.out
@@ -0,0 +1,129 @@
+Parsed test spec with 2 sessions
+
+starting permutation: s1_begin s1_create_function_in_schema s2_drop_schema s1_commit
+step s1_begin: BEGIN;
+step s1_create_function_in_schema: CREATE FUNCTION testschema.foo() RETURNS int AS 'select 1' LANGUAGE sql;
+step s2_drop_schema: DROP SCHEMA testschema; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_schema: <... completed>
+ERROR: cannot drop schema testschema because other objects depend on it
+
+starting permutation: s2_begin s2_drop_schema s1_create_function_in_schema s2_commit
+step s2_begin: BEGIN;
+step s2_drop_schema: DROP SCHEMA testschema;
+step s1_create_function_in_schema: CREATE FUNCTION testschema.foo() RETURNS int AS 'select 1' LANGUAGE sql; <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_function_in_schema: <... completed>
+ERROR: schema testschema does not exist
+
+starting permutation: s1_begin s1_alter_function_schema s2_drop_alterschema s1_commit
+step s1_begin: BEGIN;
+step s1_alter_function_schema: ALTER FUNCTION public.falter() SET SCHEMA alterschema;
+step s2_drop_alterschema: DROP SCHEMA alterschema; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_alterschema: <... completed>
+ERROR: cannot drop schema alterschema because other objects depend on it
+
+starting permutation: s2_begin s2_drop_alterschema s1_alter_function_schema s2_commit
+step s2_begin: BEGIN;
+step s2_drop_alterschema: DROP SCHEMA alterschema;
+step s1_alter_function_schema: ALTER FUNCTION public.falter() SET SCHEMA alterschema; <waiting ...>
+step s2_commit: COMMIT;
+step s1_alter_function_schema: <... completed>
+ERROR: schema alterschema does not exist
+
+starting permutation: s1_begin s1_create_function_with_argtype s2_drop_foo_type s1_commit
+step s1_begin: BEGIN;
+step s1_create_function_with_argtype: CREATE FUNCTION fooargtype(num foo) RETURNS int AS 'select 1' LANGUAGE sql;
+step s2_drop_foo_type: DROP TYPE public.foo; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_foo_type: <... completed>
+ERROR: cannot drop type foo because other objects depend on it
+
+starting permutation: s2_begin s2_drop_foo_type s1_create_function_with_argtype s2_commit
+step s2_begin: BEGIN;
+step s2_drop_foo_type: DROP TYPE public.foo;
+step s1_create_function_with_argtype: CREATE FUNCTION fooargtype(num foo) RETURNS int AS 'select 1' LANGUAGE sql; <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_function_with_argtype: <... completed>
+ERROR: type foo does not exist
+
+starting permutation: s1_begin s1_create_function_with_rettype s2_drop_foo_rettype s1_commit
+step s1_begin: BEGIN;
+step s1_create_function_with_rettype: CREATE FUNCTION footrettype() RETURNS id LANGUAGE sql RETURN 1;
+step s2_drop_foo_rettype: DROP DOMAIN id; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_foo_rettype: <... completed>
+ERROR: cannot drop type id because other objects depend on it
+
+starting permutation: s2_begin s2_drop_foo_rettype s1_create_function_with_rettype s2_commit
+step s2_begin: BEGIN;
+step s2_drop_foo_rettype: DROP DOMAIN id;
+step s1_create_function_with_rettype: CREATE FUNCTION footrettype() RETURNS id LANGUAGE sql RETURN 1; <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_function_with_rettype: <... completed>
+ERROR: type id does not exist
+
+starting permutation: s1_begin s1_create_function_with_function s2_drop_function_f s1_commit
+step s1_begin: BEGIN;
+step s1_create_function_with_function: CREATE FUNCTION foofunc() RETURNS int LANGUAGE SQL RETURN f() + 1;
+step s2_drop_function_f: DROP FUNCTION f(); <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_function_f: <... completed>
+ERROR: cannot drop function f() because other objects depend on it
+
+starting permutation: s2_begin s2_drop_function_f s1_create_function_with_function s2_commit
+step s2_begin: BEGIN;
+step s2_drop_function_f: DROP FUNCTION f();
+step s1_create_function_with_function: CREATE FUNCTION foofunc() RETURNS int LANGUAGE SQL RETURN f() + 1; <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_function_with_function: <... completed>
+ERROR: function f() does not exist
+
+starting permutation: s1_begin s1_create_domain_with_domain s2_drop_domain_id s1_commit
+step s1_begin: BEGIN;
+step s1_create_domain_with_domain: CREATE DOMAIN idid as id;
+step s2_drop_domain_id: DROP DOMAIN id; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_domain_id: <... completed>
+ERROR: cannot drop type id because other objects depend on it
+
+starting permutation: s2_begin s2_drop_domain_id s1_create_domain_with_domain s2_commit
+step s2_begin: BEGIN;
+step s2_drop_domain_id: DROP DOMAIN id;
+step s1_create_domain_with_domain: CREATE DOMAIN idid as id; <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_domain_with_domain: <... completed>
+ERROR: type id does not exist
+
+starting permutation: s1_begin s1_create_table_with_type s2_drop_footab_type s1_commit
+step s1_begin: BEGIN;
+step s1_create_table_with_type: CREATE TABLE tabtype(a footab);
+step s2_drop_footab_type: DROP TYPE public.footab; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_footab_type: <... completed>
+ERROR: cannot drop type footab because other objects depend on it
+
+starting permutation: s2_begin s2_drop_footab_type s1_create_table_with_type s2_commit
+step s2_begin: BEGIN;
+step s2_drop_footab_type: DROP TYPE public.footab;
+step s1_create_table_with_type: CREATE TABLE tabtype(a footab); <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_table_with_type: <... completed>
+ERROR: type footab does not exist
+
+starting permutation: s1_begin s1_create_server_with_fdw_wrapper s2_drop_fdw_wrapper s1_commit
+step s1_begin: BEGIN;
+step s1_create_server_with_fdw_wrapper: CREATE SERVER srv_fdw_wrapper FOREIGN DATA WRAPPER fdw_wrapper;
+step s2_drop_fdw_wrapper: DROP FOREIGN DATA WRAPPER fdw_wrapper RESTRICT; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_fdw_wrapper: <... completed>
+ERROR: cannot drop foreign-data wrapper fdw_wrapper because other objects depend on it
+
+starting permutation: s2_begin s2_drop_fdw_wrapper s1_create_server_with_fdw_wrapper s2_commit
+step s2_begin: BEGIN;
+step s2_drop_fdw_wrapper: DROP FOREIGN DATA WRAPPER fdw_wrapper RESTRICT;
+step s1_create_server_with_fdw_wrapper: CREATE SERVER srv_fdw_wrapper FOREIGN DATA WRAPPER fdw_wrapper; <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_server_with_fdw_wrapper: <... completed>
+ERROR: foreign-data wrapper fdw_wrapper does not exist
diff --git a/src/test/modules/test_dependencies_locks/meson.build b/src/test/modules/test_dependencies_locks/meson.build
new file mode 100644
index 0000000000..92a978ab93
--- /dev/null
+++ b/src/test/modules/test_dependencies_locks/meson.build
@@ -0,0 +1,12 @@
+# Copyright (c) 2024, PostgreSQL Global Development Group
+
+tests += {
+ 'name': 'test_dependencies_locks',
+ 'sd': meson.current_source_dir(),
+ 'bd': meson.current_build_dir(),
+ 'isolation': {
+ 'specs': [
+ 'test_dependencies_locks',
+ ],
+ },
+}
diff --git a/src/test/modules/test_dependencies_locks/specs/test_dependencies_locks.spec b/src/test/modules/test_dependencies_locks/specs/test_dependencies_locks.spec
new file mode 100644
index 0000000000..5d04dfe9dc
--- /dev/null
+++ b/src/test/modules/test_dependencies_locks/specs/test_dependencies_locks.spec
@@ -0,0 +1,89 @@
+setup
+{
+ CREATE SCHEMA testschema;
+ CREATE SCHEMA alterschema;
+ CREATE TYPE public.foo as enum ('one', 'two');
+ CREATE TYPE public.footab as enum ('three', 'four');
+ CREATE DOMAIN id AS int;
+ CREATE FUNCTION f() RETURNS int LANGUAGE SQL RETURN 1;
+ CREATE FUNCTION public.falter() RETURNS int LANGUAGE SQL RETURN 1;
+ CREATE FOREIGN DATA WRAPPER fdw_wrapper;
+}
+
+teardown
+{
+ DROP FUNCTION IF EXISTS testschema.foo();
+ DROP FUNCTION IF EXISTS fooargtype(num foo);
+ DROP FUNCTION IF EXISTS footrettype();
+ DROP FUNCTION IF EXISTS foofunc();
+ DROP FUNCTION IF EXISTS public.falter();
+ DROP FUNCTION IF EXISTS alterschema.falter();
+ DROP DOMAIN IF EXISTS idid;
+ DROP SERVER IF EXISTS srv_fdw_wrapper;
+ DROP TABLE IF EXISTS tabtype;
+ DROP SCHEMA IF EXISTS testschema;
+ DROP SCHEMA IF EXISTS alterschema;
+ DROP TYPE IF EXISTS public.foo;
+ DROP TYPE IF EXISTS public.footab;
+ DROP DOMAIN IF EXISTS id;
+ DROP FUNCTION IF EXISTS f();
+ DROP FOREIGN DATA WRAPPER IF EXISTS fdw_wrapper;
+}
+
+session "s1"
+
+step "s1_begin" { BEGIN; }
+step "s1_create_function_in_schema" { CREATE FUNCTION testschema.foo() RETURNS int AS 'select 1' LANGUAGE sql; }
+step "s1_create_function_with_argtype" { CREATE FUNCTION fooargtype(num foo) RETURNS int AS 'select 1' LANGUAGE sql; }
+step "s1_create_function_with_rettype" { CREATE FUNCTION footrettype() RETURNS id LANGUAGE sql RETURN 1; }
+step "s1_create_function_with_function" { CREATE FUNCTION foofunc() RETURNS int LANGUAGE SQL RETURN f() + 1; }
+step "s1_alter_function_schema" { ALTER FUNCTION public.falter() SET SCHEMA alterschema; }
+step "s1_create_domain_with_domain" { CREATE DOMAIN idid as id; }
+step "s1_create_table_with_type" { CREATE TABLE tabtype(a footab); }
+step "s1_create_server_with_fdw_wrapper" { CREATE SERVER srv_fdw_wrapper FOREIGN DATA WRAPPER fdw_wrapper; }
+step "s1_commit" { COMMIT; }
+
+session "s2"
+
+step "s2_begin" { BEGIN; }
+step "s2_drop_schema" { DROP SCHEMA testschema; }
+step "s2_drop_alterschema" { DROP SCHEMA alterschema; }
+step "s2_drop_foo_type" { DROP TYPE public.foo; }
+step "s2_drop_foo_rettype" { DROP DOMAIN id; }
+step "s2_drop_footab_type" { DROP TYPE public.footab; }
+step "s2_drop_function_f" { DROP FUNCTION f(); }
+step "s2_drop_domain_id" { DROP DOMAIN id; }
+step "s2_drop_fdw_wrapper" { DROP FOREIGN DATA WRAPPER fdw_wrapper RESTRICT; }
+step "s2_commit" { COMMIT; }
+
+# function - schema
+permutation "s1_begin" "s1_create_function_in_schema" "s2_drop_schema" "s1_commit"
+permutation "s2_begin" "s2_drop_schema" "s1_create_function_in_schema" "s2_commit"
+
+# alter function - schema
+permutation "s1_begin" "s1_alter_function_schema" "s2_drop_alterschema" "s1_commit"
+permutation "s2_begin" "s2_drop_alterschema" "s1_alter_function_schema" "s2_commit"
+
+# function - argtype
+permutation "s1_begin" "s1_create_function_with_argtype" "s2_drop_foo_type" "s1_commit"
+permutation "s2_begin" "s2_drop_foo_type" "s1_create_function_with_argtype" "s2_commit"
+
+# function - rettype
+permutation "s1_begin" "s1_create_function_with_rettype" "s2_drop_foo_rettype" "s1_commit"
+permutation "s2_begin" "s2_drop_foo_rettype" "s1_create_function_with_rettype" "s2_commit"
+
+# function - function
+permutation "s1_begin" "s1_create_function_with_function" "s2_drop_function_f" "s1_commit"
+permutation "s2_begin" "s2_drop_function_f" "s1_create_function_with_function" "s2_commit"
+
+# domain - domain
+permutation "s1_begin" "s1_create_domain_with_domain" "s2_drop_domain_id" "s1_commit"
+permutation "s2_begin" "s2_drop_domain_id" "s1_create_domain_with_domain" "s2_commit"
+
+# table - type
+permutation "s1_begin" "s1_create_table_with_type" "s2_drop_footab_type" "s1_commit"
+permutation "s2_begin" "s2_drop_footab_type" "s1_create_table_with_type" "s2_commit"
+
+# server - foreign data wrapper
+permutation "s1_begin" "s1_create_server_with_fdw_wrapper" "s2_drop_fdw_wrapper" "s1_commit"
+permutation "s2_begin" "s2_drop_fdw_wrapper" "s1_create_server_with_fdw_wrapper" "s2_commit"
--
2.34.1
^ permalink raw reply [nested|flat] 13+ messages in thread
* Re: Avoid orphaned objects dependencies, take 3
@ 2024-05-14 12:00 Alexander Lakhin <[email protected]>
parent: Bertrand Drouvot <[email protected]>
1 sibling, 1 reply; 13+ messages in thread
From: Alexander Lakhin @ 2024-05-14 12:00 UTC (permalink / raw)
To: Bertrand Drouvot <[email protected]>; +Cc: [email protected]
Hi Bertrand,
09.05.2024 15:20, Bertrand Drouvot wrote:
> Oh I see, your test updates an existing dependency. v4 took care about brand new
> dependencies creation (recordMultipleDependencies()) but forgot to take care
> about changing an existing dependency (which is done in another code path:
> changeDependencyFor()).
>
> Please find attached v5 that adds:
>
> - a call to the new depLockAndCheckObject() function in changeDependencyFor().
> - a test when altering an existing dependency.
>
> With v5 applied, I don't see the issue anymore.
Me too. Thank you for the improved version!
I will test the patch in the background, but for now I see no other
issues with it.
Best regards,
Alexander
^ permalink raw reply [nested|flat] 13+ messages in thread
* Re: Avoid orphaned objects dependencies, take 3
@ 2024-05-15 01:14 Michael Paquier <[email protected]>
parent: Bertrand Drouvot <[email protected]>
1 sibling, 1 reply; 13+ messages in thread
From: Michael Paquier @ 2024-05-15 01:14 UTC (permalink / raw)
To: Bertrand Drouvot <[email protected]>; +Cc: Alexander Lakhin <[email protected]>; [email protected]
On Thu, May 09, 2024 at 12:20:51PM +0000, Bertrand Drouvot wrote:
> Oh I see, your test updates an existing dependency. v4 took care about brand new
> dependencies creation (recordMultipleDependencies()) but forgot to take care
> about changing an existing dependency (which is done in another code path:
> changeDependencyFor()).
>
> Please find attached v5 that adds:
>
> - a call to the new depLockAndCheckObject() function in changeDependencyFor().
> - a test when altering an existing dependency.
>
> With v5 applied, I don't see the issue anymore.
+ if (object_description)
+ ereport(ERROR, errmsg("%s does not exist", object_description));
+ else
+ ereport(ERROR, errmsg("a dependent object does not ex
This generates an internal error code. Is that intended?
--- /dev/null
+++ b/src/test/modules/test_dependencies_locks/specs/test_dependencies_locks.spec
This introduces a module with only one single spec. I could get
behind an extra module if splitting the tests into more specs makes
sense or if there is a restriction on installcheck. However, for
one spec file filed with a bunch of objects, and note that I'm OK to
let this single spec grow more for this range of tests, it seems to me
that this would be better under src/test/isolation/.
+ if (use_dirty_snapshot)
+ {
+ InitDirtySnapshot(DirtySnapshot);
+ snapshot = &DirtySnapshot;
+ }
+ else
+ snapshot = NULL;
I'm wondering if Robert has a comment about that. It looks backwards
in a world where we try to encourage MVCC snapshots for catalog
scans (aka 568d4138c646), especially for the part related to
dependency.c and ObjectAddresses.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 13+ messages in thread
* Re: Avoid orphaned objects dependencies, take 3
@ 2024-05-15 08:31 Bertrand Drouvot <[email protected]>
parent: Michael Paquier <[email protected]>
0 siblings, 2 replies; 13+ messages in thread
From: Bertrand Drouvot @ 2024-05-15 08:31 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Alexander Lakhin <[email protected]>; [email protected]
Hi,
On Wed, May 15, 2024 at 10:14:09AM +0900, Michael Paquier wrote:
> On Thu, May 09, 2024 at 12:20:51PM +0000, Bertrand Drouvot wrote:
> > Oh I see, your test updates an existing dependency. v4 took care about brand new
> > dependencies creation (recordMultipleDependencies()) but forgot to take care
> > about changing an existing dependency (which is done in another code path:
> > changeDependencyFor()).
> >
> > Please find attached v5 that adds:
> >
> > - a call to the new depLockAndCheckObject() function in changeDependencyFor().
> > - a test when altering an existing dependency.
> >
> > With v5 applied, I don't see the issue anymore.
>
> + if (object_description)
> + ereport(ERROR, errmsg("%s does not exist", object_description));
> + else
> + ereport(ERROR, errmsg("a dependent object does not ex
>
> This generates an internal error code. Is that intended?
Thanks for looking at it!
Yes, it's like when say you want to create an object in a schema that does not
exist (see get_namespace_oid()).
> --- /dev/null
> +++ b/src/test/modules/test_dependencies_locks/specs/test_dependencies_locks.spec
>
> This introduces a module with only one single spec. I could get
> behind an extra module if splitting the tests into more specs makes
> sense or if there is a restriction on installcheck. However, for
> one spec file filed with a bunch of objects, and note that I'm OK to
> let this single spec grow more for this range of tests, it seems to me
> that this would be better under src/test/isolation/.
Yeah, I was not sure about this one (the location is from take 2 mentioned
up-thread). I'll look at moving the tests to src/test/isolation/.
Regards,
--
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 13+ messages in thread
* Re: Avoid orphaned objects dependencies, take 3
@ 2024-05-15 08:33 Bertrand Drouvot <[email protected]>
parent: Alexander Lakhin <[email protected]>
0 siblings, 0 replies; 13+ messages in thread
From: Bertrand Drouvot @ 2024-05-15 08:33 UTC (permalink / raw)
To: Alexander Lakhin <[email protected]>; +Cc: [email protected]
Hi,
On Tue, May 14, 2024 at 03:00:00PM +0300, Alexander Lakhin wrote:
> Hi Bertrand,
>
> 09.05.2024 15:20, Bertrand Drouvot wrote:
> > Oh I see, your test updates an existing dependency. v4 took care about brand new
> > dependencies creation (recordMultipleDependencies()) but forgot to take care
> > about changing an existing dependency (which is done in another code path:
> > changeDependencyFor()).
> >
> > Please find attached v5 that adds:
> >
> > - a call to the new depLockAndCheckObject() function in changeDependencyFor().
> > - a test when altering an existing dependency.
> >
> > With v5 applied, I don't see the issue anymore.
>
> Me too. Thank you for the improved version!
> I will test the patch in the background, but for now I see no other
> issues with it.
Thanks for confirming!
Regards,
--
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 13+ messages in thread
* Re: Avoid orphaned objects dependencies, take 3
@ 2024-05-15 10:31 Bertrand Drouvot <[email protected]>
parent: Bertrand Drouvot <[email protected]>
1 sibling, 1 reply; 13+ messages in thread
From: Bertrand Drouvot @ 2024-05-15 10:31 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Alexander Lakhin <[email protected]>; [email protected]
Hi,
On Wed, May 15, 2024 at 08:31:43AM +0000, Bertrand Drouvot wrote:
> Hi,
>
> On Wed, May 15, 2024 at 10:14:09AM +0900, Michael Paquier wrote:
> > +++ b/src/test/modules/test_dependencies_locks/specs/test_dependencies_locks.spec
> >
> > This introduces a module with only one single spec. I could get
> > behind an extra module if splitting the tests into more specs makes
> > sense or if there is a restriction on installcheck. However, for
> > one spec file filed with a bunch of objects, and note that I'm OK to
> > let this single spec grow more for this range of tests, it seems to me
> > that this would be better under src/test/isolation/.
>
> Yeah, I was not sure about this one (the location is from take 2 mentioned
> up-thread). I'll look at moving the tests to src/test/isolation/.
Please find attached v6 (only diff with v5 is moving the tests as suggested
above).
Regards,
--
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
Attachments:
[text/x-diff] v6-0001-Avoid-orphaned-objects-dependencies.patch (19.2K, ../../[email protected]/2-v6-0001-Avoid-orphaned-objects-dependencies.patch)
download | inline diff:
From 59f7befd34b8aa4ba0483a100e85bacbc1a76707 Mon Sep 17 00:00:00 2001
From: Bertrand Drouvot <[email protected]>
Date: Fri, 29 Mar 2024 15:43:26 +0000
Subject: [PATCH v6] Avoid orphaned objects dependencies
It's currently possible to create orphaned objects dependencies, for example:
Scenario 1:
session 1: begin; drop schema schem;
session 2: create a function in the schema schem
session 1: commit;
With the above, the function created in session 2 would be linked to a non
existing schema.
Scenario 2:
session 1: begin; create a function in the schema schem
session 2: drop schema schem;
session 1: commit;
With the above, the function created in session 1 would be linked to a non
existing schema.
To avoid those scenarios, a new lock (that conflicts with a lock taken by DROP)
has been put in place when the dependencies are being recorded. With this in
place, the drop schema in scenario 2 would be locked.
Also, after the new lock attempt, the patch checks that the object still exists:
with this in place session 2 in scenario 1 would be locked and would report an
error once session 1 committs (that would not be the case should session 1 abort
the transaction).
If the object is dropped before the new lock attempt is triggered then the patch
would also report an error (but with less details).
The patch adds a few tests for some dependency cases (that would currently produce
orphaned objects):
- schema and function (as the above scenarios)
- alter a dependency (function and schema)
- function and arg type
- function and return type
- function and function
- domain and domain
- table and type
- server and foreign data wrapper
---
src/backend/catalog/dependency.c | 54 ++++++++
src/backend/catalog/objectaddress.c | 70 ++++++++++
src/backend/catalog/pg_depend.c | 12 ++
src/include/catalog/dependency.h | 1 +
src/include/catalog/objectaddress.h | 1 +
.../expected/test_dependencies_locks.out | 129 ++++++++++++++++++
src/test/isolation/isolation_schedule | 1 +
.../specs/test_dependencies_locks.spec | 89 ++++++++++++
8 files changed, 357 insertions(+)
24.1% src/backend/catalog/
45.9% src/test/isolation/expected/
28.6% src/test/isolation/specs/
diff --git a/src/backend/catalog/dependency.c b/src/backend/catalog/dependency.c
index d4b5b2ade1..a49357bbe2 100644
--- a/src/backend/catalog/dependency.c
+++ b/src/backend/catalog/dependency.c
@@ -1517,6 +1517,60 @@ AcquireDeletionLock(const ObjectAddress *object, int flags)
}
}
+/*
+ * depLockAndCheckObject
+ *
+ * Lock the object that we are about to record a dependency on.
+ * After it's locked, verify that it hasn't been dropped while we
+ * weren't looking. If the object has been dropped, this function
+ * does not return!
+ */
+void
+depLockAndCheckObject(const ObjectAddress *object)
+{
+ char *object_description;
+
+ /*
+ * Those don't rely on LockDatabaseObject() when being dropped (see
+ * AcquireDeletionLock()). Also it looks like they can not produce
+ * orphaned dependent objects when being dropped.
+ */
+ if (object->classId == RelationRelationId || object->classId == AuthMemRelationId)
+ return;
+
+ object_description = getObjectDescription(object, true);
+
+ /* assume we should lock the whole object not a sub-object */
+ LockDatabaseObject(object->classId, object->objectId, 0, AccessShareLock);
+
+ /* check if object still exists */
+ if (!ObjectByIdExist(object, false))
+ {
+ /*
+ * It might be possible that we are creating it (for example creating
+ * a composite type while creating a relation), so bypass the syscache
+ * lookup and use a dirty snaphot instead to cover this scenario.
+ */
+ if (!ObjectByIdExist(object, true))
+ {
+ /*
+ * If the object has been dropped before we get a chance to get
+ * its description, then emit a generic error message. That looks
+ * like a good compromise over extra complexity.
+ */
+ if (object_description)
+ ereport(ERROR, errmsg("%s does not exist", object_description));
+ else
+ ereport(ERROR, errmsg("a dependent object does not exist"));
+ }
+ }
+
+ if (object_description)
+ pfree(object_description);
+
+ return;
+}
+
/*
* ReleaseDeletionLock - release an object deletion lock
*
diff --git a/src/backend/catalog/objectaddress.c b/src/backend/catalog/objectaddress.c
index 7b536ac6fd..c2b873dd81 100644
--- a/src/backend/catalog/objectaddress.c
+++ b/src/backend/catalog/objectaddress.c
@@ -2590,6 +2590,76 @@ get_object_namespace(const ObjectAddress *address)
return oid;
}
+/*
+ * ObjectByIdExist
+ *
+ * Return whether the given object exists.
+ *
+ * Works for most catalogs, if no special processing is needed.
+ */
+bool
+ObjectByIdExist(const ObjectAddress *address, bool use_dirty_snapshot)
+{
+ HeapTuple tuple;
+ int cache = -1;
+ const ObjectPropertyType *property;
+
+ if (!use_dirty_snapshot)
+ {
+ property = get_object_property_data(address->classId);
+
+ cache = property->oid_catcache_id;
+ }
+
+ if (cache >= 0)
+ {
+ /* Fetch tuple from syscache. */
+ tuple = SearchSysCache1(cache, ObjectIdGetDatum(address->objectId));
+
+ if (!HeapTupleIsValid(tuple))
+ {
+ return false;
+ }
+
+ ReleaseSysCache(tuple);
+
+ return true;
+ }
+ else
+ {
+ Relation rel;
+ ScanKeyData skey[1];
+ SysScanDesc scan;
+ SnapshotData DirtySnapshot;
+ Snapshot snapshot;
+
+ if (use_dirty_snapshot)
+ {
+ InitDirtySnapshot(DirtySnapshot);
+ snapshot = &DirtySnapshot;
+ }
+ else
+ snapshot = NULL;
+
+ rel = table_open(address->classId, AccessShareLock);
+
+ ScanKeyInit(&skey[0],
+ get_object_attnum_oid(address->classId),
+ BTEqualStrategyNumber, F_OIDEQ,
+ ObjectIdGetDatum(address->objectId));
+
+ scan = systable_beginscan(rel, get_object_oid_index(address->classId), true,
+ snapshot, 1, skey);
+
+ /* we expect exactly one match */
+ tuple = systable_getnext(scan);
+ systable_endscan(scan);
+ table_close(rel, AccessShareLock);
+
+ return (HeapTupleIsValid(tuple));
+ }
+}
+
/*
* Return ObjectType for the given object type as given by
* getObjectTypeDescription; if no valid ObjectType code exists, but it's a
diff --git a/src/backend/catalog/pg_depend.c b/src/backend/catalog/pg_depend.c
index 5366f7820c..f16af28429 100644
--- a/src/backend/catalog/pg_depend.c
+++ b/src/backend/catalog/pg_depend.c
@@ -108,6 +108,12 @@ recordMultipleDependencies(const ObjectAddress *depender,
if (isObjectPinned(referenced))
continue;
+ /*
+ * Acquire a lock and check object still exists while recording the
+ * dependency. XXX - Should we do so only for DEPENDENCY_NORMAL?
+ */
+ depLockAndCheckObject(referenced);
+
if (slot_init_count < max_slots)
{
slot[slot_stored_count] = MakeSingleTupleTableSlot(RelationGetDescr(dependDesc),
@@ -506,6 +512,12 @@ changeDependencyFor(Oid classId, Oid objectId,
return 1;
}
+ /*
+ * Acquire a lock and check object still exists while changing the
+ * dependency.
+ */
+ depLockAndCheckObject(&objAddr);
+
depRel = table_open(DependRelationId, RowExclusiveLock);
/* There should be existing dependency record(s), so search. */
diff --git a/src/include/catalog/dependency.h b/src/include/catalog/dependency.h
index 7eee66f810..5619b6f55e 100644
--- a/src/include/catalog/dependency.h
+++ b/src/include/catalog/dependency.h
@@ -101,6 +101,7 @@ typedef struct ObjectAddresses ObjectAddresses;
/* in dependency.c */
extern void AcquireDeletionLock(const ObjectAddress *object, int flags);
+extern void depLockAndCheckObject(const ObjectAddress *object);
extern void ReleaseDeletionLock(const ObjectAddress *object);
diff --git a/src/include/catalog/objectaddress.h b/src/include/catalog/objectaddress.h
index 3a70d80e32..04891abcc1 100644
--- a/src/include/catalog/objectaddress.h
+++ b/src/include/catalog/objectaddress.h
@@ -53,6 +53,7 @@ extern void check_object_ownership(Oid roleid,
Node *object, Relation relation);
extern Oid get_object_namespace(const ObjectAddress *address);
+extern bool ObjectByIdExist(const ObjectAddress *address, bool use_dirty_snapshot);
extern bool is_objectclass_supported(Oid class_id);
extern const char *get_object_class_descr(Oid class_id);
diff --git a/src/test/isolation/expected/test_dependencies_locks.out b/src/test/isolation/expected/test_dependencies_locks.out
new file mode 100644
index 0000000000..9b645d7aa5
--- /dev/null
+++ b/src/test/isolation/expected/test_dependencies_locks.out
@@ -0,0 +1,129 @@
+Parsed test spec with 2 sessions
+
+starting permutation: s1_begin s1_create_function_in_schema s2_drop_schema s1_commit
+step s1_begin: BEGIN;
+step s1_create_function_in_schema: CREATE FUNCTION testschema.foo() RETURNS int AS 'select 1' LANGUAGE sql;
+step s2_drop_schema: DROP SCHEMA testschema; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_schema: <... completed>
+ERROR: cannot drop schema testschema because other objects depend on it
+
+starting permutation: s2_begin s2_drop_schema s1_create_function_in_schema s2_commit
+step s2_begin: BEGIN;
+step s2_drop_schema: DROP SCHEMA testschema;
+step s1_create_function_in_schema: CREATE FUNCTION testschema.foo() RETURNS int AS 'select 1' LANGUAGE sql; <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_function_in_schema: <... completed>
+ERROR: schema testschema does not exist
+
+starting permutation: s1_begin s1_alter_function_schema s2_drop_alterschema s1_commit
+step s1_begin: BEGIN;
+step s1_alter_function_schema: ALTER FUNCTION public.falter() SET SCHEMA alterschema;
+step s2_drop_alterschema: DROP SCHEMA alterschema; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_alterschema: <... completed>
+ERROR: cannot drop schema alterschema because other objects depend on it
+
+starting permutation: s2_begin s2_drop_alterschema s1_alter_function_schema s2_commit
+step s2_begin: BEGIN;
+step s2_drop_alterschema: DROP SCHEMA alterschema;
+step s1_alter_function_schema: ALTER FUNCTION public.falter() SET SCHEMA alterschema; <waiting ...>
+step s2_commit: COMMIT;
+step s1_alter_function_schema: <... completed>
+ERROR: schema alterschema does not exist
+
+starting permutation: s1_begin s1_create_function_with_argtype s2_drop_foo_type s1_commit
+step s1_begin: BEGIN;
+step s1_create_function_with_argtype: CREATE FUNCTION fooargtype(num foo) RETURNS int AS 'select 1' LANGUAGE sql;
+step s2_drop_foo_type: DROP TYPE public.foo; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_foo_type: <... completed>
+ERROR: cannot drop type foo because other objects depend on it
+
+starting permutation: s2_begin s2_drop_foo_type s1_create_function_with_argtype s2_commit
+step s2_begin: BEGIN;
+step s2_drop_foo_type: DROP TYPE public.foo;
+step s1_create_function_with_argtype: CREATE FUNCTION fooargtype(num foo) RETURNS int AS 'select 1' LANGUAGE sql; <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_function_with_argtype: <... completed>
+ERROR: type foo does not exist
+
+starting permutation: s1_begin s1_create_function_with_rettype s2_drop_foo_rettype s1_commit
+step s1_begin: BEGIN;
+step s1_create_function_with_rettype: CREATE FUNCTION footrettype() RETURNS id LANGUAGE sql RETURN 1;
+step s2_drop_foo_rettype: DROP DOMAIN id; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_foo_rettype: <... completed>
+ERROR: cannot drop type id because other objects depend on it
+
+starting permutation: s2_begin s2_drop_foo_rettype s1_create_function_with_rettype s2_commit
+step s2_begin: BEGIN;
+step s2_drop_foo_rettype: DROP DOMAIN id;
+step s1_create_function_with_rettype: CREATE FUNCTION footrettype() RETURNS id LANGUAGE sql RETURN 1; <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_function_with_rettype: <... completed>
+ERROR: type id does not exist
+
+starting permutation: s1_begin s1_create_function_with_function s2_drop_function_f s1_commit
+step s1_begin: BEGIN;
+step s1_create_function_with_function: CREATE FUNCTION foofunc() RETURNS int LANGUAGE SQL RETURN f() + 1;
+step s2_drop_function_f: DROP FUNCTION f(); <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_function_f: <... completed>
+ERROR: cannot drop function f() because other objects depend on it
+
+starting permutation: s2_begin s2_drop_function_f s1_create_function_with_function s2_commit
+step s2_begin: BEGIN;
+step s2_drop_function_f: DROP FUNCTION f();
+step s1_create_function_with_function: CREATE FUNCTION foofunc() RETURNS int LANGUAGE SQL RETURN f() + 1; <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_function_with_function: <... completed>
+ERROR: function f() does not exist
+
+starting permutation: s1_begin s1_create_domain_with_domain s2_drop_domain_id s1_commit
+step s1_begin: BEGIN;
+step s1_create_domain_with_domain: CREATE DOMAIN idid as id;
+step s2_drop_domain_id: DROP DOMAIN id; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_domain_id: <... completed>
+ERROR: cannot drop type id because other objects depend on it
+
+starting permutation: s2_begin s2_drop_domain_id s1_create_domain_with_domain s2_commit
+step s2_begin: BEGIN;
+step s2_drop_domain_id: DROP DOMAIN id;
+step s1_create_domain_with_domain: CREATE DOMAIN idid as id; <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_domain_with_domain: <... completed>
+ERROR: type id does not exist
+
+starting permutation: s1_begin s1_create_table_with_type s2_drop_footab_type s1_commit
+step s1_begin: BEGIN;
+step s1_create_table_with_type: CREATE TABLE tabtype(a footab);
+step s2_drop_footab_type: DROP TYPE public.footab; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_footab_type: <... completed>
+ERROR: cannot drop type footab because other objects depend on it
+
+starting permutation: s2_begin s2_drop_footab_type s1_create_table_with_type s2_commit
+step s2_begin: BEGIN;
+step s2_drop_footab_type: DROP TYPE public.footab;
+step s1_create_table_with_type: CREATE TABLE tabtype(a footab); <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_table_with_type: <... completed>
+ERROR: type footab does not exist
+
+starting permutation: s1_begin s1_create_server_with_fdw_wrapper s2_drop_fdw_wrapper s1_commit
+step s1_begin: BEGIN;
+step s1_create_server_with_fdw_wrapper: CREATE SERVER srv_fdw_wrapper FOREIGN DATA WRAPPER fdw_wrapper;
+step s2_drop_fdw_wrapper: DROP FOREIGN DATA WRAPPER fdw_wrapper RESTRICT; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_fdw_wrapper: <... completed>
+ERROR: cannot drop foreign-data wrapper fdw_wrapper because other objects depend on it
+
+starting permutation: s2_begin s2_drop_fdw_wrapper s1_create_server_with_fdw_wrapper s2_commit
+step s2_begin: BEGIN;
+step s2_drop_fdw_wrapper: DROP FOREIGN DATA WRAPPER fdw_wrapper RESTRICT;
+step s1_create_server_with_fdw_wrapper: CREATE SERVER srv_fdw_wrapper FOREIGN DATA WRAPPER fdw_wrapper; <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_server_with_fdw_wrapper: <... completed>
+ERROR: foreign-data wrapper fdw_wrapper does not exist
diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule
index 0342eb39e4..1b67f0bffe 100644
--- a/src/test/isolation/isolation_schedule
+++ b/src/test/isolation/isolation_schedule
@@ -114,3 +114,4 @@ test: serializable-parallel-2
test: serializable-parallel-3
test: matview-write-skew
test: lock-nowait
+test: test_dependencies_locks
diff --git a/src/test/isolation/specs/test_dependencies_locks.spec b/src/test/isolation/specs/test_dependencies_locks.spec
new file mode 100644
index 0000000000..5d04dfe9dc
--- /dev/null
+++ b/src/test/isolation/specs/test_dependencies_locks.spec
@@ -0,0 +1,89 @@
+setup
+{
+ CREATE SCHEMA testschema;
+ CREATE SCHEMA alterschema;
+ CREATE TYPE public.foo as enum ('one', 'two');
+ CREATE TYPE public.footab as enum ('three', 'four');
+ CREATE DOMAIN id AS int;
+ CREATE FUNCTION f() RETURNS int LANGUAGE SQL RETURN 1;
+ CREATE FUNCTION public.falter() RETURNS int LANGUAGE SQL RETURN 1;
+ CREATE FOREIGN DATA WRAPPER fdw_wrapper;
+}
+
+teardown
+{
+ DROP FUNCTION IF EXISTS testschema.foo();
+ DROP FUNCTION IF EXISTS fooargtype(num foo);
+ DROP FUNCTION IF EXISTS footrettype();
+ DROP FUNCTION IF EXISTS foofunc();
+ DROP FUNCTION IF EXISTS public.falter();
+ DROP FUNCTION IF EXISTS alterschema.falter();
+ DROP DOMAIN IF EXISTS idid;
+ DROP SERVER IF EXISTS srv_fdw_wrapper;
+ DROP TABLE IF EXISTS tabtype;
+ DROP SCHEMA IF EXISTS testschema;
+ DROP SCHEMA IF EXISTS alterschema;
+ DROP TYPE IF EXISTS public.foo;
+ DROP TYPE IF EXISTS public.footab;
+ DROP DOMAIN IF EXISTS id;
+ DROP FUNCTION IF EXISTS f();
+ DROP FOREIGN DATA WRAPPER IF EXISTS fdw_wrapper;
+}
+
+session "s1"
+
+step "s1_begin" { BEGIN; }
+step "s1_create_function_in_schema" { CREATE FUNCTION testschema.foo() RETURNS int AS 'select 1' LANGUAGE sql; }
+step "s1_create_function_with_argtype" { CREATE FUNCTION fooargtype(num foo) RETURNS int AS 'select 1' LANGUAGE sql; }
+step "s1_create_function_with_rettype" { CREATE FUNCTION footrettype() RETURNS id LANGUAGE sql RETURN 1; }
+step "s1_create_function_with_function" { CREATE FUNCTION foofunc() RETURNS int LANGUAGE SQL RETURN f() + 1; }
+step "s1_alter_function_schema" { ALTER FUNCTION public.falter() SET SCHEMA alterschema; }
+step "s1_create_domain_with_domain" { CREATE DOMAIN idid as id; }
+step "s1_create_table_with_type" { CREATE TABLE tabtype(a footab); }
+step "s1_create_server_with_fdw_wrapper" { CREATE SERVER srv_fdw_wrapper FOREIGN DATA WRAPPER fdw_wrapper; }
+step "s1_commit" { COMMIT; }
+
+session "s2"
+
+step "s2_begin" { BEGIN; }
+step "s2_drop_schema" { DROP SCHEMA testschema; }
+step "s2_drop_alterschema" { DROP SCHEMA alterschema; }
+step "s2_drop_foo_type" { DROP TYPE public.foo; }
+step "s2_drop_foo_rettype" { DROP DOMAIN id; }
+step "s2_drop_footab_type" { DROP TYPE public.footab; }
+step "s2_drop_function_f" { DROP FUNCTION f(); }
+step "s2_drop_domain_id" { DROP DOMAIN id; }
+step "s2_drop_fdw_wrapper" { DROP FOREIGN DATA WRAPPER fdw_wrapper RESTRICT; }
+step "s2_commit" { COMMIT; }
+
+# function - schema
+permutation "s1_begin" "s1_create_function_in_schema" "s2_drop_schema" "s1_commit"
+permutation "s2_begin" "s2_drop_schema" "s1_create_function_in_schema" "s2_commit"
+
+# alter function - schema
+permutation "s1_begin" "s1_alter_function_schema" "s2_drop_alterschema" "s1_commit"
+permutation "s2_begin" "s2_drop_alterschema" "s1_alter_function_schema" "s2_commit"
+
+# function - argtype
+permutation "s1_begin" "s1_create_function_with_argtype" "s2_drop_foo_type" "s1_commit"
+permutation "s2_begin" "s2_drop_foo_type" "s1_create_function_with_argtype" "s2_commit"
+
+# function - rettype
+permutation "s1_begin" "s1_create_function_with_rettype" "s2_drop_foo_rettype" "s1_commit"
+permutation "s2_begin" "s2_drop_foo_rettype" "s1_create_function_with_rettype" "s2_commit"
+
+# function - function
+permutation "s1_begin" "s1_create_function_with_function" "s2_drop_function_f" "s1_commit"
+permutation "s2_begin" "s2_drop_function_f" "s1_create_function_with_function" "s2_commit"
+
+# domain - domain
+permutation "s1_begin" "s1_create_domain_with_domain" "s2_drop_domain_id" "s1_commit"
+permutation "s2_begin" "s2_drop_domain_id" "s1_create_domain_with_domain" "s2_commit"
+
+# table - type
+permutation "s1_begin" "s1_create_table_with_type" "s2_drop_footab_type" "s1_commit"
+permutation "s2_begin" "s2_drop_footab_type" "s1_create_table_with_type" "s2_commit"
+
+# server - foreign data wrapper
+permutation "s1_begin" "s1_create_server_with_fdw_wrapper" "s2_drop_fdw_wrapper" "s1_commit"
+permutation "s2_begin" "s2_drop_fdw_wrapper" "s1_create_server_with_fdw_wrapper" "s2_commit"
--
2.34.1
^ permalink raw reply [nested|flat] 13+ messages in thread
* Re: Avoid orphaned objects dependencies, take 3
@ 2024-05-19 08:00 Alexander Lakhin <[email protected]>
parent: Bertrand Drouvot <[email protected]>
1 sibling, 1 reply; 13+ messages in thread
From: Alexander Lakhin @ 2024-05-19 08:00 UTC (permalink / raw)
To: Bertrand Drouvot <[email protected]>; Michael Paquier <[email protected]>; +Cc: [email protected]
Hello Bertrand,
15.05.2024 11:31, Bertrand Drouvot wrote:
> On Wed, May 15, 2024 at 10:14:09AM +0900, Michael Paquier wrote:
>
>> + if (object_description)
>> + ereport(ERROR, errmsg("%s does not exist", object_description));
>> + else
>> + ereport(ERROR, errmsg("a dependent object does not ex
>>
>> This generates an internal error code. Is that intended?
> Yes, it's like when say you want to create an object in a schema that does not
> exist (see get_namespace_oid()).
AFAICS, get_namespace_oid() throws not ERRCODE_INTERNAL_ERROR,
but ERRCODE_UNDEFINED_SCHEMA:
# SELECT regtype('unknown_schema.type');
ERROR: schema "unknown_schema" does not exist
LINE 1: SELECT regtype('unknown_schema.type');
^
# \echo :LAST_ERROR_SQLSTATE
3F000
Probably, it's worth to avoid ERRCODE_INTERNAL_ERROR here in light of [1]
and [2], as these errors are not that abnormal (not Assert-like).
[1] https://www.postgresql.org/message-id/Zic_GNgos5sMxKoa%40paquier.xyz
[2] https://commitfest.postgresql.org/48/4735/
Best regards,
Alexander
^ permalink raw reply [nested|flat] 13+ messages in thread
* Re: Avoid orphaned objects dependencies, take 3
@ 2024-05-21 07:22 Bertrand Drouvot <[email protected]>
parent: Alexander Lakhin <[email protected]>
0 siblings, 0 replies; 13+ messages in thread
From: Bertrand Drouvot @ 2024-05-21 07:22 UTC (permalink / raw)
To: Alexander Lakhin <[email protected]>; +Cc: Michael Paquier <[email protected]>; [email protected]
Hi,
On Sun, May 19, 2024 at 11:00:00AM +0300, Alexander Lakhin wrote:
> Hello Bertrand,
>
> Probably, it's worth to avoid ERRCODE_INTERNAL_ERROR here in light of [1]
> and [2], as these errors are not that abnormal (not Assert-like).
>
> [1] https://www.postgresql.org/message-id/Zic_GNgos5sMxKoa%40paquier.xyz
> [2] https://commitfest.postgresql.org/48/4735/
>
Thanks for mentioning the above examples, I agree that it's worth to avoid
ERRCODE_INTERNAL_ERROR here: please find attached v7 that makes use of a new
ERRCODE: ERRCODE_DEPENDENT_OBJECTS_DOES_NOT_EXIST
I thought about this name as it is close enough to the already existing
"ERRCODE_DEPENDENT_OBJECTS_STILL_EXIST" but I'm open to suggestion too.
Regards,
--
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
Attachments:
[text/x-diff] v7-0001-Avoid-orphaned-objects-dependencies.patch (20.1K, ../../[email protected]/2-v7-0001-Avoid-orphaned-objects-dependencies.patch)
download | inline diff:
From 79809f88311ef1cf4e8f3250e1508fe0b7d86602 Mon Sep 17 00:00:00 2001
From: Bertrand Drouvot <[email protected]>
Date: Fri, 29 Mar 2024 15:43:26 +0000
Subject: [PATCH v7] Avoid orphaned objects dependencies
It's currently possible to create orphaned objects dependencies, for example:
Scenario 1:
session 1: begin; drop schema schem;
session 2: create a function in the schema schem
session 1: commit;
With the above, the function created in session 2 would be linked to a non
existing schema.
Scenario 2:
session 1: begin; create a function in the schema schem
session 2: drop schema schem;
session 1: commit;
With the above, the function created in session 1 would be linked to a non
existing schema.
To avoid those scenarios, a new lock (that conflicts with a lock taken by DROP)
has been put in place when the dependencies are being recorded. With this in
place, the drop schema in scenario 2 would be locked.
Also, after the new lock attempt, the patch checks that the object still exists:
with this in place session 2 in scenario 1 would be locked and would report an
error once session 1 committs (that would not be the case should session 1 abort
the transaction).
If the object is dropped before the new lock attempt is triggered then the patch
would also report an error (but with less details).
The patch adds a few tests for some dependency cases (that would currently produce
orphaned objects):
- schema and function (as the above scenarios)
- alter a dependency (function and schema)
- function and arg type
- function and return type
- function and function
- domain and domain
- table and type
- server and foreign data wrapper
---
src/backend/catalog/dependency.c | 58 ++++++++
src/backend/catalog/objectaddress.c | 70 ++++++++++
src/backend/catalog/pg_depend.c | 12 ++
src/backend/utils/errcodes.txt | 1 +
src/include/catalog/dependency.h | 1 +
src/include/catalog/objectaddress.h | 1 +
.../expected/test_dependencies_locks.out | 129 ++++++++++++++++++
src/test/isolation/isolation_schedule | 1 +
.../specs/test_dependencies_locks.spec | 89 ++++++++++++
9 files changed, 362 insertions(+)
24.6% src/backend/catalog/
45.1% src/test/isolation/expected/
28.1% src/test/isolation/specs/
diff --git a/src/backend/catalog/dependency.c b/src/backend/catalog/dependency.c
index d4b5b2ade1..89b2f18f98 100644
--- a/src/backend/catalog/dependency.c
+++ b/src/backend/catalog/dependency.c
@@ -1517,6 +1517,64 @@ AcquireDeletionLock(const ObjectAddress *object, int flags)
}
}
+/*
+ * depLockAndCheckObject
+ *
+ * Lock the object that we are about to record a dependency on.
+ * After it's locked, verify that it hasn't been dropped while we
+ * weren't looking. If the object has been dropped, this function
+ * does not return!
+ */
+void
+depLockAndCheckObject(const ObjectAddress *object)
+{
+ char *object_description;
+
+ /*
+ * Those don't rely on LockDatabaseObject() when being dropped (see
+ * AcquireDeletionLock()). Also it looks like they can not produce
+ * orphaned dependent objects when being dropped.
+ */
+ if (object->classId == RelationRelationId || object->classId == AuthMemRelationId)
+ return;
+
+ object_description = getObjectDescription(object, true);
+
+ /* assume we should lock the whole object not a sub-object */
+ LockDatabaseObject(object->classId, object->objectId, 0, AccessShareLock);
+
+ /* check if object still exists */
+ if (!ObjectByIdExist(object, false))
+ {
+ /*
+ * It might be possible that we are creating it (for example creating
+ * a composite type while creating a relation), so bypass the syscache
+ * lookup and use a dirty snaphot instead to cover this scenario.
+ */
+ if (!ObjectByIdExist(object, true))
+ {
+ /*
+ * If the object has been dropped before we get a chance to get
+ * its description, then emit a generic error message. That looks
+ * like a good compromise over extra complexity.
+ */
+ if (object_description)
+ ereport(ERROR,
+ (errcode(ERRCODE_DEPENDENT_OBJECTS_DOES_NOT_EXIST),
+ errmsg("%s does not exist", object_description)));
+ else
+ ereport(ERROR,
+ (errcode(ERRCODE_DEPENDENT_OBJECTS_DOES_NOT_EXIST),
+ errmsg("a dependent object does not exist")));
+ }
+ }
+
+ if (object_description)
+ pfree(object_description);
+
+ return;
+}
+
/*
* ReleaseDeletionLock - release an object deletion lock
*
diff --git a/src/backend/catalog/objectaddress.c b/src/backend/catalog/objectaddress.c
index 7b536ac6fd..c2b873dd81 100644
--- a/src/backend/catalog/objectaddress.c
+++ b/src/backend/catalog/objectaddress.c
@@ -2590,6 +2590,76 @@ get_object_namespace(const ObjectAddress *address)
return oid;
}
+/*
+ * ObjectByIdExist
+ *
+ * Return whether the given object exists.
+ *
+ * Works for most catalogs, if no special processing is needed.
+ */
+bool
+ObjectByIdExist(const ObjectAddress *address, bool use_dirty_snapshot)
+{
+ HeapTuple tuple;
+ int cache = -1;
+ const ObjectPropertyType *property;
+
+ if (!use_dirty_snapshot)
+ {
+ property = get_object_property_data(address->classId);
+
+ cache = property->oid_catcache_id;
+ }
+
+ if (cache >= 0)
+ {
+ /* Fetch tuple from syscache. */
+ tuple = SearchSysCache1(cache, ObjectIdGetDatum(address->objectId));
+
+ if (!HeapTupleIsValid(tuple))
+ {
+ return false;
+ }
+
+ ReleaseSysCache(tuple);
+
+ return true;
+ }
+ else
+ {
+ Relation rel;
+ ScanKeyData skey[1];
+ SysScanDesc scan;
+ SnapshotData DirtySnapshot;
+ Snapshot snapshot;
+
+ if (use_dirty_snapshot)
+ {
+ InitDirtySnapshot(DirtySnapshot);
+ snapshot = &DirtySnapshot;
+ }
+ else
+ snapshot = NULL;
+
+ rel = table_open(address->classId, AccessShareLock);
+
+ ScanKeyInit(&skey[0],
+ get_object_attnum_oid(address->classId),
+ BTEqualStrategyNumber, F_OIDEQ,
+ ObjectIdGetDatum(address->objectId));
+
+ scan = systable_beginscan(rel, get_object_oid_index(address->classId), true,
+ snapshot, 1, skey);
+
+ /* we expect exactly one match */
+ tuple = systable_getnext(scan);
+ systable_endscan(scan);
+ table_close(rel, AccessShareLock);
+
+ return (HeapTupleIsValid(tuple));
+ }
+}
+
/*
* Return ObjectType for the given object type as given by
* getObjectTypeDescription; if no valid ObjectType code exists, but it's a
diff --git a/src/backend/catalog/pg_depend.c b/src/backend/catalog/pg_depend.c
index 5366f7820c..f16af28429 100644
--- a/src/backend/catalog/pg_depend.c
+++ b/src/backend/catalog/pg_depend.c
@@ -108,6 +108,12 @@ recordMultipleDependencies(const ObjectAddress *depender,
if (isObjectPinned(referenced))
continue;
+ /*
+ * Acquire a lock and check object still exists while recording the
+ * dependency. XXX - Should we do so only for DEPENDENCY_NORMAL?
+ */
+ depLockAndCheckObject(referenced);
+
if (slot_init_count < max_slots)
{
slot[slot_stored_count] = MakeSingleTupleTableSlot(RelationGetDescr(dependDesc),
@@ -506,6 +512,12 @@ changeDependencyFor(Oid classId, Oid objectId,
return 1;
}
+ /*
+ * Acquire a lock and check object still exists while changing the
+ * dependency.
+ */
+ depLockAndCheckObject(&objAddr);
+
depRel = table_open(DependRelationId, RowExclusiveLock);
/* There should be existing dependency record(s), so search. */
diff --git a/src/backend/utils/errcodes.txt b/src/backend/utils/errcodes.txt
index 3250d539e1..60e8539fe3 100644
--- a/src/backend/utils/errcodes.txt
+++ b/src/backend/utils/errcodes.txt
@@ -271,6 +271,7 @@ Section: Class 28 - Invalid Authorization Specification
Section: Class 2B - Dependent Privilege Descriptors Still Exist
2B000 E ERRCODE_DEPENDENT_PRIVILEGE_DESCRIPTORS_STILL_EXIST dependent_privilege_descriptors_still_exist
+2BP02 E ERRCODE_DEPENDENT_OBJECTS_DOES_NOT_EXIST dependent_objects_does_not_exist
2BP01 E ERRCODE_DEPENDENT_OBJECTS_STILL_EXIST dependent_objects_still_exist
Section: Class 2D - Invalid Transaction Termination
diff --git a/src/include/catalog/dependency.h b/src/include/catalog/dependency.h
index 7eee66f810..5619b6f55e 100644
--- a/src/include/catalog/dependency.h
+++ b/src/include/catalog/dependency.h
@@ -101,6 +101,7 @@ typedef struct ObjectAddresses ObjectAddresses;
/* in dependency.c */
extern void AcquireDeletionLock(const ObjectAddress *object, int flags);
+extern void depLockAndCheckObject(const ObjectAddress *object);
extern void ReleaseDeletionLock(const ObjectAddress *object);
diff --git a/src/include/catalog/objectaddress.h b/src/include/catalog/objectaddress.h
index 3a70d80e32..04891abcc1 100644
--- a/src/include/catalog/objectaddress.h
+++ b/src/include/catalog/objectaddress.h
@@ -53,6 +53,7 @@ extern void check_object_ownership(Oid roleid,
Node *object, Relation relation);
extern Oid get_object_namespace(const ObjectAddress *address);
+extern bool ObjectByIdExist(const ObjectAddress *address, bool use_dirty_snapshot);
extern bool is_objectclass_supported(Oid class_id);
extern const char *get_object_class_descr(Oid class_id);
diff --git a/src/test/isolation/expected/test_dependencies_locks.out b/src/test/isolation/expected/test_dependencies_locks.out
new file mode 100644
index 0000000000..9b645d7aa5
--- /dev/null
+++ b/src/test/isolation/expected/test_dependencies_locks.out
@@ -0,0 +1,129 @@
+Parsed test spec with 2 sessions
+
+starting permutation: s1_begin s1_create_function_in_schema s2_drop_schema s1_commit
+step s1_begin: BEGIN;
+step s1_create_function_in_schema: CREATE FUNCTION testschema.foo() RETURNS int AS 'select 1' LANGUAGE sql;
+step s2_drop_schema: DROP SCHEMA testschema; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_schema: <... completed>
+ERROR: cannot drop schema testschema because other objects depend on it
+
+starting permutation: s2_begin s2_drop_schema s1_create_function_in_schema s2_commit
+step s2_begin: BEGIN;
+step s2_drop_schema: DROP SCHEMA testschema;
+step s1_create_function_in_schema: CREATE FUNCTION testschema.foo() RETURNS int AS 'select 1' LANGUAGE sql; <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_function_in_schema: <... completed>
+ERROR: schema testschema does not exist
+
+starting permutation: s1_begin s1_alter_function_schema s2_drop_alterschema s1_commit
+step s1_begin: BEGIN;
+step s1_alter_function_schema: ALTER FUNCTION public.falter() SET SCHEMA alterschema;
+step s2_drop_alterschema: DROP SCHEMA alterschema; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_alterschema: <... completed>
+ERROR: cannot drop schema alterschema because other objects depend on it
+
+starting permutation: s2_begin s2_drop_alterschema s1_alter_function_schema s2_commit
+step s2_begin: BEGIN;
+step s2_drop_alterschema: DROP SCHEMA alterschema;
+step s1_alter_function_schema: ALTER FUNCTION public.falter() SET SCHEMA alterschema; <waiting ...>
+step s2_commit: COMMIT;
+step s1_alter_function_schema: <... completed>
+ERROR: schema alterschema does not exist
+
+starting permutation: s1_begin s1_create_function_with_argtype s2_drop_foo_type s1_commit
+step s1_begin: BEGIN;
+step s1_create_function_with_argtype: CREATE FUNCTION fooargtype(num foo) RETURNS int AS 'select 1' LANGUAGE sql;
+step s2_drop_foo_type: DROP TYPE public.foo; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_foo_type: <... completed>
+ERROR: cannot drop type foo because other objects depend on it
+
+starting permutation: s2_begin s2_drop_foo_type s1_create_function_with_argtype s2_commit
+step s2_begin: BEGIN;
+step s2_drop_foo_type: DROP TYPE public.foo;
+step s1_create_function_with_argtype: CREATE FUNCTION fooargtype(num foo) RETURNS int AS 'select 1' LANGUAGE sql; <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_function_with_argtype: <... completed>
+ERROR: type foo does not exist
+
+starting permutation: s1_begin s1_create_function_with_rettype s2_drop_foo_rettype s1_commit
+step s1_begin: BEGIN;
+step s1_create_function_with_rettype: CREATE FUNCTION footrettype() RETURNS id LANGUAGE sql RETURN 1;
+step s2_drop_foo_rettype: DROP DOMAIN id; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_foo_rettype: <... completed>
+ERROR: cannot drop type id because other objects depend on it
+
+starting permutation: s2_begin s2_drop_foo_rettype s1_create_function_with_rettype s2_commit
+step s2_begin: BEGIN;
+step s2_drop_foo_rettype: DROP DOMAIN id;
+step s1_create_function_with_rettype: CREATE FUNCTION footrettype() RETURNS id LANGUAGE sql RETURN 1; <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_function_with_rettype: <... completed>
+ERROR: type id does not exist
+
+starting permutation: s1_begin s1_create_function_with_function s2_drop_function_f s1_commit
+step s1_begin: BEGIN;
+step s1_create_function_with_function: CREATE FUNCTION foofunc() RETURNS int LANGUAGE SQL RETURN f() + 1;
+step s2_drop_function_f: DROP FUNCTION f(); <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_function_f: <... completed>
+ERROR: cannot drop function f() because other objects depend on it
+
+starting permutation: s2_begin s2_drop_function_f s1_create_function_with_function s2_commit
+step s2_begin: BEGIN;
+step s2_drop_function_f: DROP FUNCTION f();
+step s1_create_function_with_function: CREATE FUNCTION foofunc() RETURNS int LANGUAGE SQL RETURN f() + 1; <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_function_with_function: <... completed>
+ERROR: function f() does not exist
+
+starting permutation: s1_begin s1_create_domain_with_domain s2_drop_domain_id s1_commit
+step s1_begin: BEGIN;
+step s1_create_domain_with_domain: CREATE DOMAIN idid as id;
+step s2_drop_domain_id: DROP DOMAIN id; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_domain_id: <... completed>
+ERROR: cannot drop type id because other objects depend on it
+
+starting permutation: s2_begin s2_drop_domain_id s1_create_domain_with_domain s2_commit
+step s2_begin: BEGIN;
+step s2_drop_domain_id: DROP DOMAIN id;
+step s1_create_domain_with_domain: CREATE DOMAIN idid as id; <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_domain_with_domain: <... completed>
+ERROR: type id does not exist
+
+starting permutation: s1_begin s1_create_table_with_type s2_drop_footab_type s1_commit
+step s1_begin: BEGIN;
+step s1_create_table_with_type: CREATE TABLE tabtype(a footab);
+step s2_drop_footab_type: DROP TYPE public.footab; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_footab_type: <... completed>
+ERROR: cannot drop type footab because other objects depend on it
+
+starting permutation: s2_begin s2_drop_footab_type s1_create_table_with_type s2_commit
+step s2_begin: BEGIN;
+step s2_drop_footab_type: DROP TYPE public.footab;
+step s1_create_table_with_type: CREATE TABLE tabtype(a footab); <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_table_with_type: <... completed>
+ERROR: type footab does not exist
+
+starting permutation: s1_begin s1_create_server_with_fdw_wrapper s2_drop_fdw_wrapper s1_commit
+step s1_begin: BEGIN;
+step s1_create_server_with_fdw_wrapper: CREATE SERVER srv_fdw_wrapper FOREIGN DATA WRAPPER fdw_wrapper;
+step s2_drop_fdw_wrapper: DROP FOREIGN DATA WRAPPER fdw_wrapper RESTRICT; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_fdw_wrapper: <... completed>
+ERROR: cannot drop foreign-data wrapper fdw_wrapper because other objects depend on it
+
+starting permutation: s2_begin s2_drop_fdw_wrapper s1_create_server_with_fdw_wrapper s2_commit
+step s2_begin: BEGIN;
+step s2_drop_fdw_wrapper: DROP FOREIGN DATA WRAPPER fdw_wrapper RESTRICT;
+step s1_create_server_with_fdw_wrapper: CREATE SERVER srv_fdw_wrapper FOREIGN DATA WRAPPER fdw_wrapper; <waiting ...>
+step s2_commit: COMMIT;
+step s1_create_server_with_fdw_wrapper: <... completed>
+ERROR: foreign-data wrapper fdw_wrapper does not exist
diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule
index 0342eb39e4..1b67f0bffe 100644
--- a/src/test/isolation/isolation_schedule
+++ b/src/test/isolation/isolation_schedule
@@ -114,3 +114,4 @@ test: serializable-parallel-2
test: serializable-parallel-3
test: matview-write-skew
test: lock-nowait
+test: test_dependencies_locks
diff --git a/src/test/isolation/specs/test_dependencies_locks.spec b/src/test/isolation/specs/test_dependencies_locks.spec
new file mode 100644
index 0000000000..5d04dfe9dc
--- /dev/null
+++ b/src/test/isolation/specs/test_dependencies_locks.spec
@@ -0,0 +1,89 @@
+setup
+{
+ CREATE SCHEMA testschema;
+ CREATE SCHEMA alterschema;
+ CREATE TYPE public.foo as enum ('one', 'two');
+ CREATE TYPE public.footab as enum ('three', 'four');
+ CREATE DOMAIN id AS int;
+ CREATE FUNCTION f() RETURNS int LANGUAGE SQL RETURN 1;
+ CREATE FUNCTION public.falter() RETURNS int LANGUAGE SQL RETURN 1;
+ CREATE FOREIGN DATA WRAPPER fdw_wrapper;
+}
+
+teardown
+{
+ DROP FUNCTION IF EXISTS testschema.foo();
+ DROP FUNCTION IF EXISTS fooargtype(num foo);
+ DROP FUNCTION IF EXISTS footrettype();
+ DROP FUNCTION IF EXISTS foofunc();
+ DROP FUNCTION IF EXISTS public.falter();
+ DROP FUNCTION IF EXISTS alterschema.falter();
+ DROP DOMAIN IF EXISTS idid;
+ DROP SERVER IF EXISTS srv_fdw_wrapper;
+ DROP TABLE IF EXISTS tabtype;
+ DROP SCHEMA IF EXISTS testschema;
+ DROP SCHEMA IF EXISTS alterschema;
+ DROP TYPE IF EXISTS public.foo;
+ DROP TYPE IF EXISTS public.footab;
+ DROP DOMAIN IF EXISTS id;
+ DROP FUNCTION IF EXISTS f();
+ DROP FOREIGN DATA WRAPPER IF EXISTS fdw_wrapper;
+}
+
+session "s1"
+
+step "s1_begin" { BEGIN; }
+step "s1_create_function_in_schema" { CREATE FUNCTION testschema.foo() RETURNS int AS 'select 1' LANGUAGE sql; }
+step "s1_create_function_with_argtype" { CREATE FUNCTION fooargtype(num foo) RETURNS int AS 'select 1' LANGUAGE sql; }
+step "s1_create_function_with_rettype" { CREATE FUNCTION footrettype() RETURNS id LANGUAGE sql RETURN 1; }
+step "s1_create_function_with_function" { CREATE FUNCTION foofunc() RETURNS int LANGUAGE SQL RETURN f() + 1; }
+step "s1_alter_function_schema" { ALTER FUNCTION public.falter() SET SCHEMA alterschema; }
+step "s1_create_domain_with_domain" { CREATE DOMAIN idid as id; }
+step "s1_create_table_with_type" { CREATE TABLE tabtype(a footab); }
+step "s1_create_server_with_fdw_wrapper" { CREATE SERVER srv_fdw_wrapper FOREIGN DATA WRAPPER fdw_wrapper; }
+step "s1_commit" { COMMIT; }
+
+session "s2"
+
+step "s2_begin" { BEGIN; }
+step "s2_drop_schema" { DROP SCHEMA testschema; }
+step "s2_drop_alterschema" { DROP SCHEMA alterschema; }
+step "s2_drop_foo_type" { DROP TYPE public.foo; }
+step "s2_drop_foo_rettype" { DROP DOMAIN id; }
+step "s2_drop_footab_type" { DROP TYPE public.footab; }
+step "s2_drop_function_f" { DROP FUNCTION f(); }
+step "s2_drop_domain_id" { DROP DOMAIN id; }
+step "s2_drop_fdw_wrapper" { DROP FOREIGN DATA WRAPPER fdw_wrapper RESTRICT; }
+step "s2_commit" { COMMIT; }
+
+# function - schema
+permutation "s1_begin" "s1_create_function_in_schema" "s2_drop_schema" "s1_commit"
+permutation "s2_begin" "s2_drop_schema" "s1_create_function_in_schema" "s2_commit"
+
+# alter function - schema
+permutation "s1_begin" "s1_alter_function_schema" "s2_drop_alterschema" "s1_commit"
+permutation "s2_begin" "s2_drop_alterschema" "s1_alter_function_schema" "s2_commit"
+
+# function - argtype
+permutation "s1_begin" "s1_create_function_with_argtype" "s2_drop_foo_type" "s1_commit"
+permutation "s2_begin" "s2_drop_foo_type" "s1_create_function_with_argtype" "s2_commit"
+
+# function - rettype
+permutation "s1_begin" "s1_create_function_with_rettype" "s2_drop_foo_rettype" "s1_commit"
+permutation "s2_begin" "s2_drop_foo_rettype" "s1_create_function_with_rettype" "s2_commit"
+
+# function - function
+permutation "s1_begin" "s1_create_function_with_function" "s2_drop_function_f" "s1_commit"
+permutation "s2_begin" "s2_drop_function_f" "s1_create_function_with_function" "s2_commit"
+
+# domain - domain
+permutation "s1_begin" "s1_create_domain_with_domain" "s2_drop_domain_id" "s1_commit"
+permutation "s2_begin" "s2_drop_domain_id" "s1_create_domain_with_domain" "s2_commit"
+
+# table - type
+permutation "s1_begin" "s1_create_table_with_type" "s2_drop_footab_type" "s1_commit"
+permutation "s2_begin" "s2_drop_footab_type" "s1_create_table_with_type" "s2_commit"
+
+# server - foreign data wrapper
+permutation "s1_begin" "s1_create_server_with_fdw_wrapper" "s2_drop_fdw_wrapper" "s1_commit"
+permutation "s2_begin" "s2_drop_fdw_wrapper" "s1_create_server_with_fdw_wrapper" "s2_commit"
--
2.34.1
^ permalink raw reply [nested|flat] 13+ messages in thread
* Re: Avoid orphaned objects dependencies, take 3
@ 2024-05-21 12:53 Robert Haas <[email protected]>
parent: Bertrand Drouvot <[email protected]>
0 siblings, 1 reply; 13+ messages in thread
From: Robert Haas @ 2024-05-21 12:53 UTC (permalink / raw)
To: Bertrand Drouvot <[email protected]>; +Cc: Michael Paquier <[email protected]>; Alexander Lakhin <[email protected]>; [email protected]
On Wed, May 15, 2024 at 6:31 AM Bertrand Drouvot
<[email protected]> wrote:
> Please find attached v6 (only diff with v5 is moving the tests as suggested
> above).
I don't immediately know what to think about this patch. I've known
about this issue for a long time, but I didn't think this was how we
would fix it.
I think the problem here is basically that we don't lock namespaces
(schemas) when we're adding and removing things from the schema. So I
assumed that if we ever did something about this, what we would do
would be add a bunch of calls to lock schemas to the appropriate parts
of the code. What you've done here instead is add locking at a much
lower level - whenever we are adding a dependency on an object, we
lock the object. The advantage of that approach is that we definitely
won't miss anything. The disadvantage of that approach is that it
means we have some very low-level code taking locks, which means it's
not obvious which operations are taking what locks. Maybe it could
even result in some redundancy, like the higher-level code taking a
lock also (possibly in a different mode) and then this code taking
another one.
I haven't gone through the previous threads; it sounds like there's
already been some discussion of this, but I'm just telling you how it
strikes me on first look.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 13+ messages in thread
* Re: Avoid orphaned objects dependencies, take 3
@ 2024-05-22 10:21 Bertrand Drouvot <[email protected]>
parent: Robert Haas <[email protected]>
0 siblings, 0 replies; 13+ messages in thread
From: Bertrand Drouvot @ 2024-05-22 10:21 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Michael Paquier <[email protected]>; Alexander Lakhin <[email protected]>; [email protected]
Hi,
On Tue, May 21, 2024 at 08:53:06AM -0400, Robert Haas wrote:
> On Wed, May 15, 2024 at 6:31 AM Bertrand Drouvot
> <[email protected]> wrote:
> > Please find attached v6 (only diff with v5 is moving the tests as suggested
> > above).
>
> I don't immediately know what to think about this patch.
Thanks for looking at it!
> I've known about this issue for a long time, but I didn't think this was how we
> would fix it.
I started initially with [1] but it was focusing on function-schema only.
Then I proposed [2] making use of a dirty snapshot when recording the dependency.
But this approach appeared to be "scary" and it was still failing to close
some race conditions.
Then, Tom proposed another approach in [2] which is that "creation DDL will have
to take a lock on each referenced object that'd conflict with a lock taken by DROP".
This is the one the current patch is trying to implement.
> What you've done here instead is add locking at a much
> lower level - whenever we are adding a dependency on an object, we
> lock the object. The advantage of that approach is that we definitely
> won't miss anything.
Right, as there is much more than the ones related to schemas, for example:
- function and arg type
- function and return type
- function and function
- domain and domain
- table and type
- server and foreign data wrapper
to name a few.
> The disadvantage of that approach is that it
> means we have some very low-level code taking locks, which means it's
> not obvious which operations are taking what locks.
Right, but the new operations are "only" the ones leading to recording or altering
a dependency.
> Maybe it could
> even result in some redundancy, like the higher-level code taking a
> lock also (possibly in a different mode) and then this code taking
> another one.
The one that is added here is in AccessShareLock mode. It could conflict with
the ones in AccessExclusiveLock means (If I'm not missing any):
- AcquireDeletionLock(): which is exactly what we want
- get_object_address()
- get_object_address_rv()
- ExecAlterObjectDependsStmt()
- ExecRenameStmt()
- ExecAlterObjectDependsStmt()
- ExecAlterOwnerStmt()
- RemoveObjects()
- AlterPublication()
I think there is 2 cases here:
First case: the "conflicting" lock mode is for one of our own lock then LockCheckConflicts()
would report this as a NON conflict.
Second case: the "conflicting" lock mode is NOT for one of our own lock then LockCheckConflicts()
would report a conflict. But I've the feeling that the existing code would
already lock those sessions.
One example where it would be the case:
Session 1: doing "BEGIN; ALTER FUNCTION noschemas.foo2() SET SCHEMA alterschema" would
acquire the lock in AccessExclusiveLock during ExecAlterObjectSchemaStmt()->get_object_address()->LockDatabaseObject()
(in the existing code and before the new call that would occur through changeDependencyFor()->depLockAndCheckObject()
with the patch in place).
Then, session 2: doing "alter function noschemas.foo2() owner to newrole;"
would be locked in the existing code while doing ExecAlterOwnerStmt()->get_object_address()->LockDatabaseObject()).
Means that in this example, the new lock this patch is introducing would not be
responsible of session 2 beging locked.
Was your concern about "First case" or "Second case" or both?
[1]: https://www.postgresql.org/message-id/flat/5a9daaae-5538-b209-6279-e903c3ea2157%40amazon.com
[2]: https://www.postgresql.org/message-id/flat/8369ff70-0e31-f194-2954-787f4d9e21dd%40amazon.com
Regards,
--
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 13+ messages in thread
end of thread, other threads:[~2024-05-22 10:21 UTC | newest]
Thread overview: 13+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2022-09-02 21:03 [PATCH v3 1/1] Adjust assorted hint messages that list all valid options. Nathan Bossart <[email protected]>
2024-04-25 07:20 Re: Avoid orphaned objects dependencies, take 3 Bertrand Drouvot <[email protected]>
2024-04-30 17:00 ` Re: Avoid orphaned objects dependencies, take 3 Alexander Lakhin <[email protected]>
2024-05-09 12:20 ` Re: Avoid orphaned objects dependencies, take 3 Bertrand Drouvot <[email protected]>
2024-05-14 12:00 ` Re: Avoid orphaned objects dependencies, take 3 Alexander Lakhin <[email protected]>
2024-05-15 08:33 ` Re: Avoid orphaned objects dependencies, take 3 Bertrand Drouvot <[email protected]>
2024-05-15 01:14 ` Re: Avoid orphaned objects dependencies, take 3 Michael Paquier <[email protected]>
2024-05-15 08:31 ` Re: Avoid orphaned objects dependencies, take 3 Bertrand Drouvot <[email protected]>
2024-05-15 10:31 ` Re: Avoid orphaned objects dependencies, take 3 Bertrand Drouvot <[email protected]>
2024-05-21 12:53 ` Re: Avoid orphaned objects dependencies, take 3 Robert Haas <[email protected]>
2024-05-22 10:21 ` Re: Avoid orphaned objects dependencies, take 3 Bertrand Drouvot <[email protected]>
2024-05-19 08:00 ` Re: Avoid orphaned objects dependencies, take 3 Alexander Lakhin <[email protected]>
2024-05-21 07:22 ` Re: Avoid orphaned objects dependencies, take 3 Bertrand Drouvot <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox