public inbox for [email protected]
help / color / mirror / Atom feed[PATCH v2 5/8] jit: explain: remove backend lifetime module count from function name.
55+ messages / 14 participants
[nested] [flat]
* [PATCH v2 5/8] jit: explain: remove backend lifetime module count from function name.
@ 2019-09-26 21:05 Andres Freund <[email protected]>
0 siblings, 0 replies; 55+ messages in thread
From: Andres Freund @ 2019-09-26 21:05 UTC (permalink / raw)
Also expand function name to include in which module the function is -
without that it's harder to analyze which functions were emitted
separately (a performance concern).
Author:
Reviewed-By:
Discussion: https://postgr.es/m/
Backpatch:
---
src/backend/commands/explain.c | 65 +++++++++++++++++++++++++++++-----
src/backend/jit/llvm/llvmjit.c | 18 +++++++---
src/include/jit/llvmjit.h | 5 ++-
3 files changed, 75 insertions(+), 13 deletions(-)
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 3ccb76bdfd1..02455865d9f 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -2228,6 +2228,43 @@ show_expression(Node *node, ExprState *expr, const char *qlabel,
}
}
+/*
+ * To make JIT explain output reproducible, remove the module generation from
+ * function names. That makes it a bit harder to correlate with profiles etc,
+ * but reproducability is more important.
+ */
+static char *
+jit_funcname_for_display(const char *funcname)
+{
+ int func_counter; /* nth function in query */
+ size_t mod_num; /* nth module in query */
+ size_t mod_generation; /* nth module in backend */
+ int basename_end;
+ int matchcount = 0;
+
+ /*
+ * The pattern we need to match, see llvm_expand_funcname, is
+ * "%s_%zu_%d_mod_%zu". Find the fourth _ from the end, so a _ in the name
+ * is OK.
+ */
+ for (basename_end = strlen(funcname); basename_end >= 0; basename_end--)
+ {
+ if (funcname[basename_end] == '_' && ++matchcount == 4)
+ break;
+ }
+
+ /* couldn't parse, bail out */
+ if (matchcount != 4)
+ return pstrdup(funcname);
+
+ /* couldn't parse, bail out */
+ if (sscanf(funcname + basename_end, "_%zu_%d_mod_%zu",
+ &mod_num, &func_counter, &mod_generation) != 3)
+ return pstrdup(funcname);
+
+ return psprintf("%s_%zu_%d", pnstrdup(funcname, basename_end), mod_num, func_counter);
+}
+
static void
show_jit_expr_details(ExprState *expr, ExplainState *es)
{
@@ -2239,7 +2276,8 @@ show_jit_expr_details(ExprState *expr, ExplainState *es)
if (es->format == EXPLAIN_FORMAT_TEXT)
{
if (expr->flags & EEO_FLAG_JIT_EXPR)
- appendStringInfo(es->str, "JIT-Expr: %s", expr->expr_funcname);
+ appendStringInfo(es->str, "JIT-Expr: %s",
+ jit_funcname_for_display(expr->expr_funcname));
else
appendStringInfoString(es->str, "JIT-Expr: false");
@@ -2250,19 +2288,22 @@ show_jit_expr_details(ExprState *expr, ExplainState *es)
*/
if (expr->scan_funcname)
- appendStringInfo(es->str, ", JIT-Deform-Scan: %s", expr->scan_funcname);
+ appendStringInfo(es->str, ", JIT-Deform-Scan: %s",
+ jit_funcname_for_display(expr->scan_funcname));
else if (expr->flags & EEO_FLAG_JIT_EXPR &&
expr->flags & EEO_FLAG_DEFORM_SCAN)
appendStringInfo(es->str, ", JIT-Deform-Scan: false");
if (expr->outer_funcname)
- appendStringInfo(es->str, ", JIT-Deform-Outer: %s", expr->outer_funcname);
+ appendStringInfo(es->str, ", JIT-Deform-Outer: %s",
+ jit_funcname_for_display(expr->outer_funcname));
else if (expr->flags & EEO_FLAG_JIT_EXPR &&
expr->flags & EEO_FLAG_DEFORM_OUTER)
appendStringInfo(es->str, ", JIT-Deform-Outer: false");
if (expr->inner_funcname)
- appendStringInfo(es->str, ", JIT-Deform-Inner: %s", expr->inner_funcname);
+ appendStringInfo(es->str, ", JIT-Deform-Inner: %s",
+ jit_funcname_for_display(expr->inner_funcname));
else if (expr->flags & EEO_FLAG_JIT_EXPR &&
expr->flags & (EEO_FLAG_DEFORM_INNER))
appendStringInfo(es->str, ", JIT-Deform-Inner: false");
@@ -2270,26 +2311,34 @@ show_jit_expr_details(ExprState *expr, ExplainState *es)
else
{
if (expr->flags & EEO_FLAG_JIT_EXPR)
- ExplainPropertyText("JIT-Expr", expr->expr_funcname, es);
+ ExplainPropertyText("JIT-Expr",
+ jit_funcname_for_display(expr->expr_funcname),
+ es);
else
ExplainPropertyBool("JIT-Expr", false, es);
if (expr->scan_funcname)
- ExplainProperty("JIT-Deform-Scan", NULL, expr->scan_funcname, false, es);
+ ExplainProperty("JIT-Deform-Scan", NULL,
+ jit_funcname_for_display(expr->scan_funcname),
+ false, es);
else if (expr->flags & EEO_FLAG_DEFORM_SCAN)
ExplainProperty("JIT-Deform-Scan", NULL, "false", true, es);
else
ExplainProperty("JIT-Deform-Scan", NULL, "null", true, es);
if (expr->outer_funcname)
- ExplainProperty("JIT-Deform-Outer", NULL, expr->outer_funcname, false, es);
+ ExplainProperty("JIT-Deform-Outer", NULL,
+ jit_funcname_for_display(expr->outer_funcname),
+ false, es);
else if (expr->flags & EEO_FLAG_DEFORM_OUTER)
ExplainProperty("JIT-Deform-Outer", NULL, "false", true, es);
else
ExplainProperty("JIT-Deform-Outer", NULL, "null", true, es);
if (expr->inner_funcname)
- ExplainProperty("JIT-Deform-Inner", NULL, expr->inner_funcname, false, es);
+ ExplainProperty("JIT-Deform-Inner", NULL,
+ jit_funcname_for_display(expr->inner_funcname),
+ false, es);
else if (expr->flags & EEO_FLAG_DEFORM_INNER)
ExplainProperty("JIT-Deform-Inner", NULL, "false", true, es);
else
diff --git a/src/backend/jit/llvm/llvmjit.c b/src/backend/jit/llvm/llvmjit.c
index 5489e118041..177a00f3826 100644
--- a/src/backend/jit/llvm/llvmjit.c
+++ b/src/backend/jit/llvm/llvmjit.c
@@ -227,6 +227,8 @@ llvm_mutable_module(LLVMJitContext *context)
char *
llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
{
+ char *funcname;
+
Assert(context->module != NULL);
context->base.instr.created_functions++;
@@ -234,11 +236,19 @@ llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
/*
* Previously we used dots to separate, but turns out some tools, e.g.
* GDB, don't like that and truncate name.
+ *
+ * Append the backend-lifetime module count to the end, so it's easier for
+ * humans and machines to compare the generated function names across
+ * queries, the prefix will be the same from query execution to query
+ * execution.
*/
- return psprintf("%s_%zu_%d",
- basename,
- context->module_generation,
- context->counter++);
+ funcname = psprintf("%s_%zu_%d_mod_%zu",
+ basename,
+ context->base.instr.created_modules - 1,
+ context->counter++,
+ context->module_generation);
+
+ return funcname;
}
/*
diff --git a/src/include/jit/llvmjit.h b/src/include/jit/llvmjit.h
index 6178864b2e6..e45ff99194f 100644
--- a/src/include/jit/llvmjit.h
+++ b/src/include/jit/llvmjit.h
@@ -41,7 +41,10 @@ typedef struct LLVMJitContext
{
JitContext base;
- /* number of modules created */
+ /*
+ * llvm_generation when ->module was created, monotonically increasing
+ * within the lifetime of a backend.
+ */
size_t module_generation;
/* current, "open for write", module */
--
2.23.0.385.gbc12974a89
--ga6shgqrocqphdjc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0006-WIP-explain-Show-per-phase-information-about-aggr.patch"
^ permalink raw reply [nested|flat] 55+ messages in thread
* [PATCH v1 05/12] jit: explain: remove backend lifetime module count from function name.
@ 2019-09-26 21:05 Andres Freund <[email protected]>
0 siblings, 0 replies; 55+ messages in thread
From: Andres Freund @ 2019-09-26 21:05 UTC (permalink / raw)
Also expand function name to include in which module the function is -
without that it's harder to analyze which functions were emitted
separately (a performance concern).
Author:
Reviewed-By:
Discussion: https://postgr.es/m/
Backpatch:
---
src/backend/commands/explain.c | 65 +++++++++++++++++++++++++++++-----
src/backend/jit/llvm/llvmjit.c | 18 +++++++---
src/include/jit/llvmjit.h | 5 ++-
3 files changed, 75 insertions(+), 13 deletions(-)
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 3ccb76bdfd1..02455865d9f 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -2228,6 +2228,43 @@ show_expression(Node *node, ExprState *expr, const char *qlabel,
}
}
+/*
+ * To make JIT explain output reproducible, remove the module generation from
+ * function names. That makes it a bit harder to correlate with profiles etc,
+ * but reproducability is more important.
+ */
+static char *
+jit_funcname_for_display(const char *funcname)
+{
+ int func_counter; /* nth function in query */
+ size_t mod_num; /* nth module in query */
+ size_t mod_generation; /* nth module in backend */
+ int basename_end;
+ int matchcount = 0;
+
+ /*
+ * The pattern we need to match, see llvm_expand_funcname, is
+ * "%s_%zu_%d_mod_%zu". Find the fourth _ from the end, so a _ in the name
+ * is OK.
+ */
+ for (basename_end = strlen(funcname); basename_end >= 0; basename_end--)
+ {
+ if (funcname[basename_end] == '_' && ++matchcount == 4)
+ break;
+ }
+
+ /* couldn't parse, bail out */
+ if (matchcount != 4)
+ return pstrdup(funcname);
+
+ /* couldn't parse, bail out */
+ if (sscanf(funcname + basename_end, "_%zu_%d_mod_%zu",
+ &mod_num, &func_counter, &mod_generation) != 3)
+ return pstrdup(funcname);
+
+ return psprintf("%s_%zu_%d", pnstrdup(funcname, basename_end), mod_num, func_counter);
+}
+
static void
show_jit_expr_details(ExprState *expr, ExplainState *es)
{
@@ -2239,7 +2276,8 @@ show_jit_expr_details(ExprState *expr, ExplainState *es)
if (es->format == EXPLAIN_FORMAT_TEXT)
{
if (expr->flags & EEO_FLAG_JIT_EXPR)
- appendStringInfo(es->str, "JIT-Expr: %s", expr->expr_funcname);
+ appendStringInfo(es->str, "JIT-Expr: %s",
+ jit_funcname_for_display(expr->expr_funcname));
else
appendStringInfoString(es->str, "JIT-Expr: false");
@@ -2250,19 +2288,22 @@ show_jit_expr_details(ExprState *expr, ExplainState *es)
*/
if (expr->scan_funcname)
- appendStringInfo(es->str, ", JIT-Deform-Scan: %s", expr->scan_funcname);
+ appendStringInfo(es->str, ", JIT-Deform-Scan: %s",
+ jit_funcname_for_display(expr->scan_funcname));
else if (expr->flags & EEO_FLAG_JIT_EXPR &&
expr->flags & EEO_FLAG_DEFORM_SCAN)
appendStringInfo(es->str, ", JIT-Deform-Scan: false");
if (expr->outer_funcname)
- appendStringInfo(es->str, ", JIT-Deform-Outer: %s", expr->outer_funcname);
+ appendStringInfo(es->str, ", JIT-Deform-Outer: %s",
+ jit_funcname_for_display(expr->outer_funcname));
else if (expr->flags & EEO_FLAG_JIT_EXPR &&
expr->flags & EEO_FLAG_DEFORM_OUTER)
appendStringInfo(es->str, ", JIT-Deform-Outer: false");
if (expr->inner_funcname)
- appendStringInfo(es->str, ", JIT-Deform-Inner: %s", expr->inner_funcname);
+ appendStringInfo(es->str, ", JIT-Deform-Inner: %s",
+ jit_funcname_for_display(expr->inner_funcname));
else if (expr->flags & EEO_FLAG_JIT_EXPR &&
expr->flags & (EEO_FLAG_DEFORM_INNER))
appendStringInfo(es->str, ", JIT-Deform-Inner: false");
@@ -2270,26 +2311,34 @@ show_jit_expr_details(ExprState *expr, ExplainState *es)
else
{
if (expr->flags & EEO_FLAG_JIT_EXPR)
- ExplainPropertyText("JIT-Expr", expr->expr_funcname, es);
+ ExplainPropertyText("JIT-Expr",
+ jit_funcname_for_display(expr->expr_funcname),
+ es);
else
ExplainPropertyBool("JIT-Expr", false, es);
if (expr->scan_funcname)
- ExplainProperty("JIT-Deform-Scan", NULL, expr->scan_funcname, false, es);
+ ExplainProperty("JIT-Deform-Scan", NULL,
+ jit_funcname_for_display(expr->scan_funcname),
+ false, es);
else if (expr->flags & EEO_FLAG_DEFORM_SCAN)
ExplainProperty("JIT-Deform-Scan", NULL, "false", true, es);
else
ExplainProperty("JIT-Deform-Scan", NULL, "null", true, es);
if (expr->outer_funcname)
- ExplainProperty("JIT-Deform-Outer", NULL, expr->outer_funcname, false, es);
+ ExplainProperty("JIT-Deform-Outer", NULL,
+ jit_funcname_for_display(expr->outer_funcname),
+ false, es);
else if (expr->flags & EEO_FLAG_DEFORM_OUTER)
ExplainProperty("JIT-Deform-Outer", NULL, "false", true, es);
else
ExplainProperty("JIT-Deform-Outer", NULL, "null", true, es);
if (expr->inner_funcname)
- ExplainProperty("JIT-Deform-Inner", NULL, expr->inner_funcname, false, es);
+ ExplainProperty("JIT-Deform-Inner", NULL,
+ jit_funcname_for_display(expr->inner_funcname),
+ false, es);
else if (expr->flags & EEO_FLAG_DEFORM_INNER)
ExplainProperty("JIT-Deform-Inner", NULL, "false", true, es);
else
diff --git a/src/backend/jit/llvm/llvmjit.c b/src/backend/jit/llvm/llvmjit.c
index 5489e118041..177a00f3826 100644
--- a/src/backend/jit/llvm/llvmjit.c
+++ b/src/backend/jit/llvm/llvmjit.c
@@ -227,6 +227,8 @@ llvm_mutable_module(LLVMJitContext *context)
char *
llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
{
+ char *funcname;
+
Assert(context->module != NULL);
context->base.instr.created_functions++;
@@ -234,11 +236,19 @@ llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
/*
* Previously we used dots to separate, but turns out some tools, e.g.
* GDB, don't like that and truncate name.
+ *
+ * Append the backend-lifetime module count to the end, so it's easier for
+ * humans and machines to compare the generated function names across
+ * queries, the prefix will be the same from query execution to query
+ * execution.
*/
- return psprintf("%s_%zu_%d",
- basename,
- context->module_generation,
- context->counter++);
+ funcname = psprintf("%s_%zu_%d_mod_%zu",
+ basename,
+ context->base.instr.created_modules - 1,
+ context->counter++,
+ context->module_generation);
+
+ return funcname;
}
/*
diff --git a/src/include/jit/llvmjit.h b/src/include/jit/llvmjit.h
index 6178864b2e6..e45ff99194f 100644
--- a/src/include/jit/llvmjit.h
+++ b/src/include/jit/llvmjit.h
@@ -41,7 +41,10 @@ typedef struct LLVMJitContext
{
JitContext base;
- /* number of modules created */
+ /*
+ * llvm_generation when ->module was created, monotonically increasing
+ * within the lifetime of a backend.
+ */
size_t module_generation;
/* current, "open for write", module */
--
2.23.0.162.gf1d4a28250
--kgppz6muakthis74
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v1-0006-WIP-explain-Show-per-phase-information-about-aggr.patch"
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2024-10-10 15:54 Artur Zakirov <[email protected]>
0 siblings, 1 reply; 55+ messages in thread
From: Artur Zakirov @ 2024-10-10 15:54 UTC (permalink / raw)
To: Alexander Korotkov <[email protected]>; +Cc: Andrei Lepikhov <[email protected]>; Alexander Lakhin <[email protected]>; Pavel Borisov <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
Hi all,
On Fri, 13 Sept 2024 at 01:38, Alexander Korotkov <[email protected]> wrote:
>
> 0001 - adds comment about concurrent invalidation handling
> 0002 - revised c14d4acb8. Now we track type oids, whose
> TypeCacheEntry's filing is in-progress. Add entry to
> RelIdToTypeIdCacheHash at the end of lookup_type_cache() or on the
> transaction abort. During invalidation don't assert
> RelIdToTypeIdCacheHash to be here if TypeCacheEntry is in-progress.
Thank you Alexander for the patch. I reviewed and tested it.
I used Teodor's script to check the performance. On my laptop on
master ROLLBACK runs 11496.219 ms. With patch ROLLBACK runs 378.990
ms.
It seems to me that there are couple of possible issues in the patch:
In `lookup_type_cache()` `in_progress_list` is allocated using
`CacheMemoryContext`, on the other hand it seems there might be a case
when `CacheMemoryContext` is not created yet. It is created below in
the code if it doesn't exist:
/* Also make sure CacheMemoryContext exists */
if (!CacheMemoryContext)
CreateCacheMemoryContext();
It is probably a very rare case, but it might be better to allocate
`in_progress_list` after that line, or move creation of
`CacheMemoryContext` higher.
Within `insert_rel_type_cache_if_needed()` and
`delete_rel_type_cache_if_needed()` there is an if condition:
if ((typentry->flags & TCFLAGS_HAVE_PG_TYPE_DATA) ||
(typentry->flags & TCFLAGS_OPERATOR_FLAGS) ||
typentry->tupDesc != NULL)
Based on the logic of the rest of the code does it make sense to use
TCFLAGS_DOMAIN_BASE_IS_COMPOSITE instead of TCFLAGS_OPERATOR_FLAGS?
Otherwise the condition doesn't look logical.
--
Kind regards,
Artur
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2024-10-13 12:08 Alexander Korotkov <[email protected]>
parent: Artur Zakirov <[email protected]>
0 siblings, 1 reply; 55+ messages in thread
From: Alexander Korotkov @ 2024-10-13 12:08 UTC (permalink / raw)
To: Artur Zakirov <[email protected]>; +Cc: Andrei Lepikhov <[email protected]>; Alexander Lakhin <[email protected]>; Pavel Borisov <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
Hi, Arthur!
Thank you so much for your review!
On Thu, Oct 10, 2024 at 6:54 PM Artur Zakirov <[email protected]> wrote:
> On Fri, 13 Sept 2024 at 01:38, Alexander Korotkov <[email protected]> wrote:
> >
> > 0001 - adds comment about concurrent invalidation handling
> > 0002 - revised c14d4acb8. Now we track type oids, whose
> > TypeCacheEntry's filing is in-progress. Add entry to
> > RelIdToTypeIdCacheHash at the end of lookup_type_cache() or on the
> > transaction abort. During invalidation don't assert
> > RelIdToTypeIdCacheHash to be here if TypeCacheEntry is in-progress.
>
> Thank you Alexander for the patch. I reviewed and tested it.
>
> I used Teodor's script to check the performance. On my laptop on
> master ROLLBACK runs 11496.219 ms. With patch ROLLBACK runs 378.990
> ms.
>
> It seems to me that there are couple of possible issues in the patch:
>
> In `lookup_type_cache()` `in_progress_list` is allocated using
> `CacheMemoryContext`, on the other hand it seems there might be a case
> when `CacheMemoryContext` is not created yet. It is created below in
> the code if it doesn't exist:
>
> /* Also make sure CacheMemoryContext exists */
> if (!CacheMemoryContext)
> CreateCacheMemoryContext();
>
> It is probably a very rare case, but it might be better to allocate
> `in_progress_list` after that line, or move creation of
> `CacheMemoryContext` higher.
Yes, it makes sense to initialize `in_progress_list ` when
`CacheMemoryContext` is guaranteed to be initialized. Fixed in the
attached patch.
> Within `insert_rel_type_cache_if_needed()` and
> `delete_rel_type_cache_if_needed()` there is an if condition:
>
> if ((typentry->flags & TCFLAGS_HAVE_PG_TYPE_DATA) ||
> (typentry->flags & TCFLAGS_OPERATOR_FLAGS) ||
> typentry->tupDesc != NULL)
>
> Based on the logic of the rest of the code does it make sense to use
> TCFLAGS_DOMAIN_BASE_IS_COMPOSITE instead of TCFLAGS_OPERATOR_FLAGS?
> Otherwise the condition doesn't look logical.
I'm not sure I get the point. This check ensures that type entry has
something to be cleared. In this case we need to keep
RelIdToTypeIdCacheHash entry to find this item on invalidation
message. I'm not sure how TCFLAGS_DOMAIN_BASE_IS_COMPOSITE is
relevant here, because it's valid only for TYPTYPE_DOMAIN while this
patch deals with TYPTYPE_COMPOSITE.
------
Regards,
Alexander Korotkov
Supabase
Attachments:
[application/octet-stream] v12-0002-Avoid-looping-over-all-type-cache-entries-in-Typ.patch (18.0K, ../../CAPpHfdtQHpr6aMALxRmB+6SGBsYWVdTim9iW+zJtUjjU1L2CPQ@mail.gmail.com/2-v12-0002-Avoid-looping-over-all-type-cache-entries-in-Typ.patch)
download | inline diff:
From ace04f1edd1f71300175738dd60f85570b1c646f Mon Sep 17 00:00:00 2001
From: Alexander Korotkov <[email protected]>
Date: Tue, 10 Sep 2024 23:25:04 +0300
Subject: [PATCH v12 2/2] Avoid looping over all type cache entries in
TypeCacheRelCallback()
Currently when a single relcache entry gets invalidated,
TypeCacheRelCallback() has to loop over all type cache entries to find
appropriate typentry to invalidate. Unfortunately, using the syscache here
is impossible, because this callback could be called outside a transaction
and this makes impossible catalog lookups. This is why present commit
introduces RelIdToTypeIdCacheHash to map relation OID to its composite type
OID.
We are keeping RelIdToTypeIdCacheHash entry while corresponding type cache
entry have something to clean. Therefore, RelIdToTypeIdCacheHash shouldn't
get bloat in the case of temporary tables flood.
Discussion: https://postgr.es/m/5812a6e5-68ae-4d84-9d85-b443176966a1%40sigaev.ru
Author: Teodor Sigaev
Reviewed-by: Aleksander Alekseev, Tom Lane, Michael Paquier, Roman Zharkov
Reviewed-by: Andrei Lepikhov, Pavel Borisov
---
src/backend/access/transam/xact.c | 10 +
src/backend/utils/cache/typcache.c | 350 +++++++++++++++++++++++++----
src/include/utils/typcache.h | 4 +
src/tools/pgindent/typedefs.list | 1 +
4 files changed, 321 insertions(+), 44 deletions(-)
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 87700c7c5c7..b0b05e28790 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -70,6 +70,7 @@
#include "utils/snapmgr.h"
#include "utils/timeout.h"
#include "utils/timestamp.h"
+#include "utils/typcache.h"
/*
* User-tweakable parameters
@@ -2407,6 +2408,9 @@ CommitTransaction(void)
/* Clean up the relation cache */
AtEOXact_RelationCache(true);
+ /* Clean up the type cache */
+ AtEOXact_TypeCache();
+
/*
* Make catalog changes visible to all backends. This has to happen after
* relcache references are dropped (see comments for
@@ -2709,6 +2713,9 @@ PrepareTransaction(void)
/* Clean up the relation cache */
AtEOXact_RelationCache(true);
+ /* Clean up the type cache */
+ AtEOXact_TypeCache();
+
/* notify doesn't need a postprepare call */
PostPrepare_PgStat();
@@ -2951,6 +2958,7 @@ AbortTransaction(void)
false, true);
AtEOXact_Buffers(false);
AtEOXact_RelationCache(false);
+ AtEOXact_TypeCache();
AtEOXact_Inval(false);
AtEOXact_MultiXact();
ResourceOwnerRelease(TopTransactionResourceOwner,
@@ -5153,6 +5161,7 @@ CommitSubTransaction(void)
true, false);
AtEOSubXact_RelationCache(true, s->subTransactionId,
s->parent->subTransactionId);
+ AtEOSubXact_TypeCache();
AtEOSubXact_Inval(true);
AtSubCommit_smgr();
@@ -5328,6 +5337,7 @@ AbortSubTransaction(void)
AtEOSubXact_RelationCache(false, s->subTransactionId,
s->parent->subTransactionId);
+ AtEOSubXact_TypeCache();
AtEOSubXact_Inval(false);
ResourceOwnerRelease(s->curTransactionOwner,
RESOURCE_RELEASE_LOCKS,
diff --git a/src/backend/utils/cache/typcache.c b/src/backend/utils/cache/typcache.c
index 11382547ec0..fe37ff442b9 100644
--- a/src/backend/utils/cache/typcache.c
+++ b/src/backend/utils/cache/typcache.c
@@ -77,6 +77,20 @@
/* The main type cache hashtable searched by lookup_type_cache */
static HTAB *TypeCacheHash = NULL;
+/*
+ * The mapping of relation's OID to the corresponding composite type OID.
+ * We're keeping the map entry when the corresponding typentry has something
+ * to clear i.e it has either TCFLAGS_HAVE_PG_TYPE_DATA, or
+ * TCFLAGS_OPERATOR_FLAGS, or tupdesc.
+ */
+static HTAB *RelIdToTypeIdCacheHash = NULL;
+
+typedef struct RelIdToTypeIdCacheEntry
+{
+ Oid relid; /* OID of the relation */
+ Oid composite_typid; /* OID of the relation's composite type */
+} RelIdToTypeIdCacheEntry;
+
/* List of type cache entries for domain types */
static TypeCacheEntry *firstDomainTypeEntry = NULL;
@@ -208,6 +222,10 @@ typedef struct SharedTypmodTableEntry
dsa_pointer shared_tupdesc;
} SharedTypmodTableEntry;
+static Oid *in_progress_list;
+static int in_progress_list_len;
+static int in_progress_list_maxlen;
+
/*
* A comparator function for SharedRecordTableKey.
*/
@@ -329,6 +347,8 @@ static void shared_record_typmod_registry_detach(dsm_segment *segment,
static TupleDesc find_or_make_matching_shared_tupledesc(TupleDesc tupdesc);
static dsa_pointer share_tupledesc(dsa_area *area, TupleDesc tupdesc,
uint32 typmod);
+static void insert_rel_type_cache_if_needed(TypeCacheEntry *typentry);
+static void delete_rel_type_cache_if_needed(TypeCacheEntry *typentry);
/*
@@ -366,11 +386,13 @@ lookup_type_cache(Oid type_id, int flags)
{
TypeCacheEntry *typentry;
bool found;
+ int in_progress_offset;
if (TypeCacheHash == NULL)
{
/* First time through: initialize the hash table */
HASHCTL ctl;
+ int allocsize;
ctl.keysize = sizeof(Oid);
ctl.entrysize = sizeof(TypeCacheEntry);
@@ -385,6 +407,13 @@ lookup_type_cache(Oid type_id, int flags)
TypeCacheHash = hash_create("Type information cache", 64,
&ctl, HASH_ELEM | HASH_FUNCTION);
+ Assert(RelIdToTypeIdCacheHash == NULL);
+
+ ctl.keysize = sizeof(Oid);
+ ctl.entrysize = sizeof(RelIdToTypeIdCacheEntry);
+ RelIdToTypeIdCacheHash = hash_create("Map from relid to OID of cached composite type", 64,
+ &ctl, HASH_ELEM | HASH_BLOBS);
+
/* Also set up callbacks for SI invalidations */
CacheRegisterRelcacheCallback(TypeCacheRelCallback, (Datum) 0);
CacheRegisterSyscacheCallback(TYPEOID, TypeCacheTypCallback, (Datum) 0);
@@ -394,8 +423,32 @@ lookup_type_cache(Oid type_id, int flags)
/* Also make sure CacheMemoryContext exists */
if (!CacheMemoryContext)
CreateCacheMemoryContext();
+
+ /*
+ * reserve enough in_progress_list slots for many cases
+ */
+ allocsize = 4;
+ in_progress_list =
+ MemoryContextAlloc(CacheMemoryContext,
+ allocsize * sizeof(*in_progress_list));
+ in_progress_list_maxlen = allocsize;
}
+ Assert(TypeCacheHash != NULL && RelIdToTypeIdCacheHash != NULL);
+
+ /* Register to catch invalidation messages */
+ if (in_progress_list_len >= in_progress_list_maxlen)
+ {
+ int allocsize;
+
+ allocsize = in_progress_list_maxlen * 2;
+ in_progress_list = repalloc(in_progress_list,
+ allocsize * sizeof(*in_progress_list));
+ in_progress_list_maxlen = allocsize;
+ }
+ in_progress_offset = in_progress_list_len++;
+ in_progress_list[in_progress_offset] = type_id;
+
/* Try to look up an existing entry */
typentry = (TypeCacheEntry *) hash_search(TypeCacheHash,
&type_id,
@@ -896,6 +949,11 @@ lookup_type_cache(Oid type_id, int flags)
load_domaintype_info(typentry);
}
+ Assert(in_progress_offset + 1 == in_progress_list_len);
+ in_progress_list_len--;
+
+ insert_rel_type_cache_if_needed(typentry);
+
return typentry;
}
@@ -2290,6 +2348,53 @@ SharedRecordTypmodRegistryAttach(SharedRecordTypmodRegistry *registry)
CurrentSession->shared_typmod_table = typmod_table;
}
+/*
+ * InvalidateCompositeTypeCacheEntry
+ * Invalidate particular TypeCacheEntry on Relcache inval callback
+ *
+ * Delete the cached tuple descriptor (if any) for the given composite
+ * type, and reset whatever info we have cached about the composite type's
+ * comparability.
+ */
+static void
+InvalidateCompositeTypeCacheEntry(TypeCacheEntry *typentry)
+{
+ bool hadTupDescOrOpclass;
+
+ Assert(typentry->typtype == TYPTYPE_COMPOSITE &&
+ OidIsValid(typentry->typrelid));
+
+ hadTupDescOrOpclass = (typentry->tupDesc != NULL) ||
+ (typentry->flags & TCFLAGS_OPERATOR_FLAGS);
+
+ /* Delete tupdesc if we have it */
+ if (typentry->tupDesc != NULL)
+ {
+ /*
+ * Release our refcount and free the tupdesc if none remain. We can't
+ * use DecrTupleDescRefCount here because this reference is not logged
+ * by the current resource owner.
+ */
+ Assert(typentry->tupDesc->tdrefcount > 0);
+ if (--typentry->tupDesc->tdrefcount == 0)
+ FreeTupleDesc(typentry->tupDesc);
+ typentry->tupDesc = NULL;
+
+ /*
+ * Also clear tupDesc_identifier, so that anyone watching it will
+ * realize that the tupdesc has changed.
+ */
+ typentry->tupDesc_identifier = 0;
+ }
+
+ /* Reset equality/comparison/hashing validity information */
+ typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
+
+ /* Call check_delete_rel_type_cache() if we actually cleared something */
+ if (hadTupDescOrOpclass)
+ delete_rel_type_cache_if_needed(typentry);
+}
+
/*
* TypeCacheRelCallback
* Relcache inval callback function
@@ -2299,63 +2404,55 @@ SharedRecordTypmodRegistryAttach(SharedRecordTypmodRegistry *registry)
* whatever info we have cached about the composite type's comparability.
*
* This is called when a relcache invalidation event occurs for the given
- * relid. We must scan the whole typcache hash since we don't know the
- * type OID corresponding to the relid. We could do a direct search if this
- * were a syscache-flush callback on pg_type, but then we would need all
- * ALTER-TABLE-like commands that could modify a rowtype to issue syscache
- * invals against the rel's pg_type OID. The extra SI signaling could very
- * well cost more than we'd save, since in most usages there are not very
- * many entries in a backend's typcache. The risk of bugs-of-omission seems
- * high, too.
- *
- * Another possibility, with only localized impact, is to maintain a second
- * hashtable that indexes composite-type typcache entries by their typrelid.
- * But it's still not clear it's worth the trouble.
+ * relid. We can't use syscache to find a type corresponding to the given
+ * relation because the code can be called outside of transaction. Thus we use
+ * the RelIdToTypeIdCacheHash map to locate appropriate typcache entry.
*/
static void
TypeCacheRelCallback(Datum arg, Oid relid)
{
- HASH_SEQ_STATUS status;
TypeCacheEntry *typentry;
- /* TypeCacheHash must exist, else this callback wouldn't be registered */
- hash_seq_init(&status, TypeCacheHash);
- while ((typentry = (TypeCacheEntry *) hash_seq_search(&status)) != NULL)
+ /*
+ * RelIdToTypeIdCacheHash and TypeCacheHash should exist, otherwise this
+ * callback wouldn't be registered
+ */
+ if (OidIsValid(relid))
{
- if (typentry->typtype == TYPTYPE_COMPOSITE)
+ RelIdToTypeIdCacheEntry *relentry;
+
+ /*
+ * Find an RelIdToTypeIdCacheHash entry, which should exist as soon as
+ * corresponding typcache entry has something to clean.
+ */
+ relentry = (RelIdToTypeIdCacheEntry *) hash_search(RelIdToTypeIdCacheHash,
+ &relid,
+ HASH_FIND, NULL);
+
+ if (relentry != NULL)
{
- /* Skip if no match, unless we're zapping all composite types */
- if (relid != typentry->typrelid && relid != InvalidOid)
- continue;
+ typentry = (TypeCacheEntry *) hash_search(TypeCacheHash,
+ &relentry->composite_typid,
+ HASH_FIND, NULL);
- /* Delete tupdesc if we have it */
- if (typentry->tupDesc != NULL)
+ if (typentry != NULL)
{
- /*
- * Release our refcount, and free the tupdesc if none remain.
- * (Can't use DecrTupleDescRefCount because this reference is
- * not logged in current resource owner.)
- */
- Assert(typentry->tupDesc->tdrefcount > 0);
- if (--typentry->tupDesc->tdrefcount == 0)
- FreeTupleDesc(typentry->tupDesc);
- typentry->tupDesc = NULL;
+ Assert(typentry->typtype == TYPTYPE_COMPOSITE);
+ Assert(relid == typentry->typrelid);
- /*
- * Also clear tupDesc_identifier, so that anything watching
- * that will realize that the tupdesc has possibly changed.
- * (Alternatively, we could specify that to detect possible
- * tupdesc change, one must check for tupDesc != NULL as well
- * as tupDesc_identifier being the same as what was previously
- * seen. That seems error-prone.)
- */
- typentry->tupDesc_identifier = 0;
+ InvalidateCompositeTypeCacheEntry(typentry);
}
-
- /* Reset equality/comparison/hashing validity information */
- typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
}
- else if (typentry->typtype == TYPTYPE_DOMAIN)
+
+ /*
+ * Visit all the domain types sequentially. Typically this shouldn't
+ * affect performance since domain types are less tended to bloat.
+ * Domain types are created manually, unlike composite types which are
+ * automatically created for every temporary table.
+ */
+ for (typentry = firstDomainTypeEntry;
+ typentry != NULL;
+ typentry = typentry->nextDomain)
{
/*
* If it's domain over composite, reset flags. (We don't bother
@@ -2367,6 +2464,36 @@ TypeCacheRelCallback(Datum arg, Oid relid)
typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
}
}
+ else
+ {
+ HASH_SEQ_STATUS status;
+
+ /*
+ * Relid is invalid. By convention we need to reset all composite
+ * types in cache. Also, we should reset flags for domain types, and
+ * we loop over all entries in hash, so, do it in a single scan.
+ */
+ hash_seq_init(&status, TypeCacheHash);
+ while ((typentry = (TypeCacheEntry *) hash_seq_search(&status)) != NULL)
+ {
+ if (typentry->typtype == TYPTYPE_COMPOSITE)
+ {
+ InvalidateCompositeTypeCacheEntry(typentry);
+ }
+ else if (typentry->typtype == TYPTYPE_DOMAIN)
+ {
+ /*
+ * If it's domain over composite, reset flags. (We don't
+ * bother trying to determine whether the specific base type
+ * needs a reset.) Note that if we haven't determined whether
+ * the base type is composite, we don't need to reset
+ * anything.
+ */
+ if (typentry->flags & TCFLAGS_DOMAIN_BASE_IS_COMPOSITE)
+ typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
+ }
+ }
+ }
}
/*
@@ -2397,6 +2524,8 @@ TypeCacheTypCallback(Datum arg, int cacheid, uint32 hashvalue)
while ((typentry = (TypeCacheEntry *) hash_seq_search(&status)) != NULL)
{
+ bool hadPgTypeData = (typentry->flags & TCFLAGS_HAVE_PG_TYPE_DATA);
+
Assert(hashvalue == 0 || typentry->type_id_hash == hashvalue);
/*
@@ -2406,6 +2535,13 @@ TypeCacheTypCallback(Datum arg, int cacheid, uint32 hashvalue)
*/
typentry->flags &= ~(TCFLAGS_HAVE_PG_TYPE_DATA |
TCFLAGS_CHECKED_DOMAIN_CONSTRAINTS);
+
+ /*
+ * Call check_delete_rel_type_cache() if we cleaned
+ * TCFLAGS_HAVE_PG_TYPE_DATA flag previously.
+ */
+ if (hadPgTypeData)
+ delete_rel_type_cache_if_needed(typentry);
}
}
@@ -2914,3 +3050,129 @@ shared_record_typmod_registry_detach(dsm_segment *segment, Datum datum)
}
CurrentSession->shared_typmod_registry = NULL;
}
+
+/*
+ * Insert RelIdToTypeIdCacheHash entry if needed.
+ */
+static void
+insert_rel_type_cache_if_needed(TypeCacheEntry *typentry)
+{
+ /* Immediately quit for non-composite types */
+ if (typentry->typtype != TYPTYPE_COMPOSITE)
+ return;
+
+ /* typrelid should be given for composite types */
+ Assert(OidIsValid(typentry->typrelid));
+
+ /*
+ * Insert a RelIdToTypeIdCacheHash entry if the typentry have any
+ * information indicating it should be here.
+ */
+ if ((typentry->flags & TCFLAGS_HAVE_PG_TYPE_DATA) ||
+ (typentry->flags & TCFLAGS_OPERATOR_FLAGS) ||
+ typentry->tupDesc != NULL)
+ {
+ RelIdToTypeIdCacheEntry *relentry;
+ bool found;
+
+ relentry = (RelIdToTypeIdCacheEntry *) hash_search(RelIdToTypeIdCacheHash,
+ &typentry->typrelid,
+ HASH_ENTER, &found);
+ relentry->relid = typentry->typrelid;
+ relentry->composite_typid = typentry->type_id;
+ }
+}
+
+/*
+ * Delete entry RelIdToTypeIdCacheHash if needed after resetting of the
+ * TCFLAGS_HAVE_PG_TYPE_DATA flag, or any of TCFLAGS_OPERATOR_FLAGS flags,
+ * or tupDesc.
+ */
+static void
+delete_rel_type_cache_if_needed(TypeCacheEntry *typentry)
+{
+#ifdef USE_ASSERT_CHECKING
+ int i;
+ bool is_in_progress = false;
+
+ for (i = 0; i < in_progress_list_len; i++)
+ {
+ if (in_progress_list[i] == typentry->type_id)
+ {
+ is_in_progress = true;
+ break;
+ }
+ }
+#endif
+
+ /* Immediately quit for non-composite types */
+ if (typentry->typtype != TYPTYPE_COMPOSITE)
+ return;
+
+ /* typrelid should be given for composite types */
+ Assert(OidIsValid(typentry->typrelid));
+
+ /*
+ * Delete a RelIdToTypeIdCacheHash entry if the typentry doesn't have any
+ * information indicating entry should be still there.
+ */
+ if (!(typentry->flags & TCFLAGS_HAVE_PG_TYPE_DATA) &&
+ !(typentry->flags & TCFLAGS_OPERATOR_FLAGS) &&
+ typentry->tupDesc == NULL)
+ {
+ bool found;
+
+ (void) hash_search(RelIdToTypeIdCacheHash,
+ &typentry->typrelid,
+ HASH_REMOVE, &found);
+ Assert(found || is_in_progress);
+ }
+ else
+ {
+#ifdef USE_ASSERT_CHECKING
+ /*
+ * In assert-enabled builds otherwise check for RelIdToTypeIdCacheHash
+ * entry if it should exist.
+ */
+ bool found;
+
+ if (!is_in_progress)
+ {
+ (void) hash_search(RelIdToTypeIdCacheHash,
+ &typentry->typrelid,
+ HASH_FIND, &found);
+ Assert(found);
+ }
+#endif
+ }
+}
+
+static void
+cleanup_in_progress_typentries(void)
+{
+ int i;
+
+ for (i = 0; i < in_progress_list_len; i++)
+ {
+ TypeCacheEntry *typentry;
+
+ typentry = (TypeCacheEntry *) hash_search(TypeCacheHash,
+ &in_progress_list[i],
+ HASH_FIND, NULL);
+ insert_rel_type_cache_if_needed(typentry);
+ }
+
+ in_progress_list_len = 0;
+}
+
+void
+AtEOXact_TypeCache(void)
+{
+ cleanup_in_progress_typentries();
+}
+
+void
+AtEOSubXact_TypeCache(void)
+{
+ cleanup_in_progress_typentries();
+}
diff --git a/src/include/utils/typcache.h b/src/include/utils/typcache.h
index f506cc4aa35..f3d73ecee3a 100644
--- a/src/include/utils/typcache.h
+++ b/src/include/utils/typcache.h
@@ -207,4 +207,8 @@ extern void SharedRecordTypmodRegistryInit(SharedRecordTypmodRegistry *,
extern void SharedRecordTypmodRegistryAttach(SharedRecordTypmodRegistry *);
+extern void AtEOXact_TypeCache(void);
+
+extern void AtEOSubXact_TypeCache(void);
+
#endif /* TYPCACHE_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index a65e1c07c5d..80c25099e67 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2378,6 +2378,7 @@ RelFileLocator
RelFileLocatorBackend
RelFileNumber
RelIdCacheEnt
+RelIdToTypeIdCacheEntry
RelInfo
RelInfoArr
RelMapFile
--
2.39.5 (Apple Git-154)
[application/octet-stream] v12-0001-Update-header-comment-for-lookup_type_cache.patch (1.4K, ../../CAPpHfdtQHpr6aMALxRmB+6SGBsYWVdTim9iW+zJtUjjU1L2CPQ@mail.gmail.com/3-v12-0001-Update-header-comment-for-lookup_type_cache.patch)
download | inline diff:
From 92bbee11fa999ce9ad89349ab174f9e48ba2c753 Mon Sep 17 00:00:00 2001
From: Alexander Korotkov <[email protected]>
Date: Fri, 13 Sep 2024 02:10:04 +0300
Subject: [PATCH v12 1/2] Update header comment for lookup_type_cache()
Describe the way we handle concurrent invalidation messages.
---
src/backend/utils/cache/typcache.c | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/src/backend/utils/cache/typcache.c b/src/backend/utils/cache/typcache.c
index 2ec136b7d30..11382547ec0 100644
--- a/src/backend/utils/cache/typcache.c
+++ b/src/backend/utils/cache/typcache.c
@@ -351,6 +351,15 @@ type_cache_syshash(const void *key, Size keysize)
* invalid. Note however that we may fail to find one or more of the
* values requested by 'flags'; the caller needs to check whether the fields
* are InvalidOid or not.
+ *
+ * Note that while filling TypeCacheEntry we might process concurrent
+ * invalidation messages, causing our not-yet-filled TypeCacheEntry to be
+ * invalidated. In this case, we typically only clear flags while values are
+ * still available for the caller. It's expected that the caller holds
+ * enough locks on type-depending objects that the values are still relevant.
+ * It's also important that the tupdesc is filled in the last for
+ * TYPTYPE_COMPOSITE. So, it can't get invalidated during the
+ * lookup_type_cache() call.
*/
TypeCacheEntry *
lookup_type_cache(Oid type_id, int flags)
--
2.39.5 (Apple Git-154)
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2024-10-15 07:34 jian he <[email protected]>
parent: Alexander Korotkov <[email protected]>
0 siblings, 1 reply; 55+ messages in thread
From: jian he @ 2024-10-15 07:34 UTC (permalink / raw)
To: Alexander Korotkov <[email protected]>; +Cc: Artur Zakirov <[email protected]>; Andrei Lepikhov <[email protected]>; Alexander Lakhin <[email protected]>; Pavel Borisov <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
On Sun, Oct 13, 2024 at 8:09 PM Alexander Korotkov <[email protected]> wrote:
>
hi. Alexander.
I don't fully understand all of it. but I did some tests anyway.
static void
cleanup_in_progress_typentries(void)
{
int i;
if (in_progress_list_len > 1)
elog(INFO, "%s:%d in_progress_list_len > 1", __FILE_NAME__, __LINE__);
for (i = 0; i < in_progress_list_len; i++)
{
TypeCacheEntry *typentry;
typentry = (TypeCacheEntry *) hash_search(TypeCacheHash,
&in_progress_list[i],
HASH_FIND, NULL);
insert_rel_type_cache_if_needed(typentry);
}
in_progress_list_len = 0;
}
the regress still passed.
I assume "elog(INFO, " won't interfere in cleanup_in_progress_typentries.
So we lack tests for larger in_progress_list_len values or i missed something?
/* Call check_delete_rel_type_cache() if we actually cleared something */
if (hadTupDescOrOpclass)
delete_rel_type_cache_if_needed(typentry);
/*
* Call check_delete_rel_type_cache() if we cleaned
* TCFLAGS_HAVE_PG_TYPE_DATA flag previously.
*/
if (hadPgTypeData)
delete_rel_type_cache_if_needed(typentry);
check_delete_rel_type_cache don't exist, so these comments are wrong?
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2024-10-15 08:08 Alexander Korotkov <[email protected]>
parent: jian he <[email protected]>
0 siblings, 3 replies; 55+ messages in thread
From: Alexander Korotkov @ 2024-10-15 08:08 UTC (permalink / raw)
To: jian he <[email protected]>; +Cc: Artur Zakirov <[email protected]>; Andrei Lepikhov <[email protected]>; Alexander Lakhin <[email protected]>; Pavel Borisov <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
Hi, Jian!
Thank you for your review.
On Tue, Oct 15, 2024 at 10:34 AM jian he <[email protected]> wrote:
> I don't fully understand all of it. but I did some tests anyway.
>
> static void
> cleanup_in_progress_typentries(void)
> {
> int i;
> if (in_progress_list_len > 1)
> elog(INFO, "%s:%d in_progress_list_len > 1", __FILE_NAME__, __LINE__);
> for (i = 0; i < in_progress_list_len; i++)
> {
> TypeCacheEntry *typentry;
> typentry = (TypeCacheEntry *) hash_search(TypeCacheHash,
> &in_progress_list[i],
> HASH_FIND, NULL);
> insert_rel_type_cache_if_needed(typentry);
> }
> in_progress_list_len = 0;
> }
>
> the regress still passed.
> I assume "elog(INFO, " won't interfere in cleanup_in_progress_typentries.
> So we lack tests for larger in_progress_list_len values or i missed something?
Try to run test suite with -DCLOBBER_CACHE_ALWAYS.
> /* Call check_delete_rel_type_cache() if we actually cleared something */
> if (hadTupDescOrOpclass)
> delete_rel_type_cache_if_needed(typentry);
>
> /*
> * Call check_delete_rel_type_cache() if we cleaned
> * TCFLAGS_HAVE_PG_TYPE_DATA flag previously.
> */
> if (hadPgTypeData)
> delete_rel_type_cache_if_needed(typentry);
>
> check_delete_rel_type_cache don't exist, so these comments are wrong?
Yep, they didn't get updated. Fixed in the attached patchset.
------
Regards,
Alexander Korotkov
Supabase
Attachments:
[application/octet-stream] v13-0001-Update-header-comment-for-lookup_type_cache.patch (1.4K, ../../CAPpHfdss4xOrAXD8bS+0nc2GyZBuNX=1QbgLwpjCLg4rzbbrhw@mail.gmail.com/2-v13-0001-Update-header-comment-for-lookup_type_cache.patch)
download | inline diff:
From 8e736ebc3f69fec323351bd466d178309b734e27 Mon Sep 17 00:00:00 2001
From: Alexander Korotkov <[email protected]>
Date: Fri, 13 Sep 2024 02:10:04 +0300
Subject: [PATCH v13 1/2] Update header comment for lookup_type_cache()
Describe the way we handle concurrent invalidation messages.
---
src/backend/utils/cache/typcache.c | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/src/backend/utils/cache/typcache.c b/src/backend/utils/cache/typcache.c
index 2ec136b7d30..11382547ec0 100644
--- a/src/backend/utils/cache/typcache.c
+++ b/src/backend/utils/cache/typcache.c
@@ -351,6 +351,15 @@ type_cache_syshash(const void *key, Size keysize)
* invalid. Note however that we may fail to find one or more of the
* values requested by 'flags'; the caller needs to check whether the fields
* are InvalidOid or not.
+ *
+ * Note that while filling TypeCacheEntry we might process concurrent
+ * invalidation messages, causing our not-yet-filled TypeCacheEntry to be
+ * invalidated. In this case, we typically only clear flags while values are
+ * still available for the caller. It's expected that the caller holds
+ * enough locks on type-depending objects that the values are still relevant.
+ * It's also important that the tupdesc is filled in the last for
+ * TYPTYPE_COMPOSITE. So, it can't get invalidated during the
+ * lookup_type_cache() call.
*/
TypeCacheEntry *
lookup_type_cache(Oid type_id, int flags)
--
2.39.5 (Apple Git-154)
[application/octet-stream] v13-0002-Avoid-looping-over-all-type-cache-entries-in-Typ.patch (18.0K, ../../CAPpHfdss4xOrAXD8bS+0nc2GyZBuNX=1QbgLwpjCLg4rzbbrhw@mail.gmail.com/3-v13-0002-Avoid-looping-over-all-type-cache-entries-in-Typ.patch)
download | inline diff:
From d2fe600b042ea6b21f4a2460b4754b47b3775e8e Mon Sep 17 00:00:00 2001
From: Alexander Korotkov <[email protected]>
Date: Tue, 10 Sep 2024 23:25:04 +0300
Subject: [PATCH v13 2/2] Avoid looping over all type cache entries in
TypeCacheRelCallback()
Currently when a single relcache entry gets invalidated,
TypeCacheRelCallback() has to loop over all type cache entries to find
appropriate typentry to invalidate. Unfortunately, using the syscache here
is impossible, because this callback could be called outside a transaction
and this makes impossible catalog lookups. This is why present commit
introduces RelIdToTypeIdCacheHash to map relation OID to its composite type
OID.
We are keeping RelIdToTypeIdCacheHash entry while corresponding type cache
entry have something to clean. Therefore, RelIdToTypeIdCacheHash shouldn't
get bloat in the case of temporary tables flood.
Discussion: https://postgr.es/m/5812a6e5-68ae-4d84-9d85-b443176966a1%40sigaev.ru
Author: Teodor Sigaev
Reviewed-by: Aleksander Alekseev, Tom Lane, Michael Paquier, Roman Zharkov
Reviewed-by: Andrei Lepikhov, Pavel Borisov
---
src/backend/access/transam/xact.c | 10 +
src/backend/utils/cache/typcache.c | 350 +++++++++++++++++++++++++----
src/include/utils/typcache.h | 4 +
src/tools/pgindent/typedefs.list | 1 +
4 files changed, 321 insertions(+), 44 deletions(-)
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 87700c7c5c7..b0b05e28790 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -70,6 +70,7 @@
#include "utils/snapmgr.h"
#include "utils/timeout.h"
#include "utils/timestamp.h"
+#include "utils/typcache.h"
/*
* User-tweakable parameters
@@ -2407,6 +2408,9 @@ CommitTransaction(void)
/* Clean up the relation cache */
AtEOXact_RelationCache(true);
+ /* Clean up the type cache */
+ AtEOXact_TypeCache();
+
/*
* Make catalog changes visible to all backends. This has to happen after
* relcache references are dropped (see comments for
@@ -2709,6 +2713,9 @@ PrepareTransaction(void)
/* Clean up the relation cache */
AtEOXact_RelationCache(true);
+ /* Clean up the type cache */
+ AtEOXact_TypeCache();
+
/* notify doesn't need a postprepare call */
PostPrepare_PgStat();
@@ -2951,6 +2958,7 @@ AbortTransaction(void)
false, true);
AtEOXact_Buffers(false);
AtEOXact_RelationCache(false);
+ AtEOXact_TypeCache();
AtEOXact_Inval(false);
AtEOXact_MultiXact();
ResourceOwnerRelease(TopTransactionResourceOwner,
@@ -5153,6 +5161,7 @@ CommitSubTransaction(void)
true, false);
AtEOSubXact_RelationCache(true, s->subTransactionId,
s->parent->subTransactionId);
+ AtEOSubXact_TypeCache();
AtEOSubXact_Inval(true);
AtSubCommit_smgr();
@@ -5328,6 +5337,7 @@ AbortSubTransaction(void)
AtEOSubXact_RelationCache(false, s->subTransactionId,
s->parent->subTransactionId);
+ AtEOSubXact_TypeCache();
AtEOSubXact_Inval(false);
ResourceOwnerRelease(s->curTransactionOwner,
RESOURCE_RELEASE_LOCKS,
diff --git a/src/backend/utils/cache/typcache.c b/src/backend/utils/cache/typcache.c
index 11382547ec0..f54e7d531a8 100644
--- a/src/backend/utils/cache/typcache.c
+++ b/src/backend/utils/cache/typcache.c
@@ -77,6 +77,20 @@
/* The main type cache hashtable searched by lookup_type_cache */
static HTAB *TypeCacheHash = NULL;
+/*
+ * The mapping of relation's OID to the corresponding composite type OID.
+ * We're keeping the map entry when the corresponding typentry has something
+ * to clear i.e it has either TCFLAGS_HAVE_PG_TYPE_DATA, or
+ * TCFLAGS_OPERATOR_FLAGS, or tupdesc.
+ */
+static HTAB *RelIdToTypeIdCacheHash = NULL;
+
+typedef struct RelIdToTypeIdCacheEntry
+{
+ Oid relid; /* OID of the relation */
+ Oid composite_typid; /* OID of the relation's composite type */
+} RelIdToTypeIdCacheEntry;
+
/* List of type cache entries for domain types */
static TypeCacheEntry *firstDomainTypeEntry = NULL;
@@ -208,6 +222,10 @@ typedef struct SharedTypmodTableEntry
dsa_pointer shared_tupdesc;
} SharedTypmodTableEntry;
+static Oid *in_progress_list;
+static int in_progress_list_len;
+static int in_progress_list_maxlen;
+
/*
* A comparator function for SharedRecordTableKey.
*/
@@ -329,6 +347,8 @@ static void shared_record_typmod_registry_detach(dsm_segment *segment,
static TupleDesc find_or_make_matching_shared_tupledesc(TupleDesc tupdesc);
static dsa_pointer share_tupledesc(dsa_area *area, TupleDesc tupdesc,
uint32 typmod);
+static void insert_rel_type_cache_if_needed(TypeCacheEntry *typentry);
+static void delete_rel_type_cache_if_needed(TypeCacheEntry *typentry);
/*
@@ -366,11 +386,13 @@ lookup_type_cache(Oid type_id, int flags)
{
TypeCacheEntry *typentry;
bool found;
+ int in_progress_offset;
if (TypeCacheHash == NULL)
{
/* First time through: initialize the hash table */
HASHCTL ctl;
+ int allocsize;
ctl.keysize = sizeof(Oid);
ctl.entrysize = sizeof(TypeCacheEntry);
@@ -385,6 +407,13 @@ lookup_type_cache(Oid type_id, int flags)
TypeCacheHash = hash_create("Type information cache", 64,
&ctl, HASH_ELEM | HASH_FUNCTION);
+ Assert(RelIdToTypeIdCacheHash == NULL);
+
+ ctl.keysize = sizeof(Oid);
+ ctl.entrysize = sizeof(RelIdToTypeIdCacheEntry);
+ RelIdToTypeIdCacheHash = hash_create("Map from relid to OID of cached composite type", 64,
+ &ctl, HASH_ELEM | HASH_BLOBS);
+
/* Also set up callbacks for SI invalidations */
CacheRegisterRelcacheCallback(TypeCacheRelCallback, (Datum) 0);
CacheRegisterSyscacheCallback(TYPEOID, TypeCacheTypCallback, (Datum) 0);
@@ -394,8 +423,32 @@ lookup_type_cache(Oid type_id, int flags)
/* Also make sure CacheMemoryContext exists */
if (!CacheMemoryContext)
CreateCacheMemoryContext();
+
+ /*
+ * reserve enough in_progress_list slots for many cases
+ */
+ allocsize = 4;
+ in_progress_list =
+ MemoryContextAlloc(CacheMemoryContext,
+ allocsize * sizeof(*in_progress_list));
+ in_progress_list_maxlen = allocsize;
}
+ Assert(TypeCacheHash != NULL && RelIdToTypeIdCacheHash != NULL);
+
+ /* Register to catch invalidation messages */
+ if (in_progress_list_len >= in_progress_list_maxlen)
+ {
+ int allocsize;
+
+ allocsize = in_progress_list_maxlen * 2;
+ in_progress_list = repalloc(in_progress_list,
+ allocsize * sizeof(*in_progress_list));
+ in_progress_list_maxlen = allocsize;
+ }
+ in_progress_offset = in_progress_list_len++;
+ in_progress_list[in_progress_offset] = type_id;
+
/* Try to look up an existing entry */
typentry = (TypeCacheEntry *) hash_search(TypeCacheHash,
&type_id,
@@ -896,6 +949,11 @@ lookup_type_cache(Oid type_id, int flags)
load_domaintype_info(typentry);
}
+ Assert(in_progress_offset + 1 == in_progress_list_len);
+ in_progress_list_len--;
+
+ insert_rel_type_cache_if_needed(typentry);
+
return typentry;
}
@@ -2290,6 +2348,53 @@ SharedRecordTypmodRegistryAttach(SharedRecordTypmodRegistry *registry)
CurrentSession->shared_typmod_table = typmod_table;
}
+/*
+ * InvalidateCompositeTypeCacheEntry
+ * Invalidate particular TypeCacheEntry on Relcache inval callback
+ *
+ * Delete the cached tuple descriptor (if any) for the given composite
+ * type, and reset whatever info we have cached about the composite type's
+ * comparability.
+ */
+static void
+InvalidateCompositeTypeCacheEntry(TypeCacheEntry *typentry)
+{
+ bool hadTupDescOrOpclass;
+
+ Assert(typentry->typtype == TYPTYPE_COMPOSITE &&
+ OidIsValid(typentry->typrelid));
+
+ hadTupDescOrOpclass = (typentry->tupDesc != NULL) ||
+ (typentry->flags & TCFLAGS_OPERATOR_FLAGS);
+
+ /* Delete tupdesc if we have it */
+ if (typentry->tupDesc != NULL)
+ {
+ /*
+ * Release our refcount and free the tupdesc if none remain. We can't
+ * use DecrTupleDescRefCount here because this reference is not logged
+ * by the current resource owner.
+ */
+ Assert(typentry->tupDesc->tdrefcount > 0);
+ if (--typentry->tupDesc->tdrefcount == 0)
+ FreeTupleDesc(typentry->tupDesc);
+ typentry->tupDesc = NULL;
+
+ /*
+ * Also clear tupDesc_identifier, so that anyone watching it will
+ * realize that the tupdesc has changed.
+ */
+ typentry->tupDesc_identifier = 0;
+ }
+
+ /* Reset equality/comparison/hashing validity information */
+ typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
+
+ /* Call delete_rel_type_cache() if we actually cleared something */
+ if (hadTupDescOrOpclass)
+ delete_rel_type_cache_if_needed(typentry);
+}
+
/*
* TypeCacheRelCallback
* Relcache inval callback function
@@ -2299,63 +2404,55 @@ SharedRecordTypmodRegistryAttach(SharedRecordTypmodRegistry *registry)
* whatever info we have cached about the composite type's comparability.
*
* This is called when a relcache invalidation event occurs for the given
- * relid. We must scan the whole typcache hash since we don't know the
- * type OID corresponding to the relid. We could do a direct search if this
- * were a syscache-flush callback on pg_type, but then we would need all
- * ALTER-TABLE-like commands that could modify a rowtype to issue syscache
- * invals against the rel's pg_type OID. The extra SI signaling could very
- * well cost more than we'd save, since in most usages there are not very
- * many entries in a backend's typcache. The risk of bugs-of-omission seems
- * high, too.
- *
- * Another possibility, with only localized impact, is to maintain a second
- * hashtable that indexes composite-type typcache entries by their typrelid.
- * But it's still not clear it's worth the trouble.
+ * relid. We can't use syscache to find a type corresponding to the given
+ * relation because the code can be called outside of transaction. Thus we use
+ * the RelIdToTypeIdCacheHash map to locate appropriate typcache entry.
*/
static void
TypeCacheRelCallback(Datum arg, Oid relid)
{
- HASH_SEQ_STATUS status;
TypeCacheEntry *typentry;
- /* TypeCacheHash must exist, else this callback wouldn't be registered */
- hash_seq_init(&status, TypeCacheHash);
- while ((typentry = (TypeCacheEntry *) hash_seq_search(&status)) != NULL)
+ /*
+ * RelIdToTypeIdCacheHash and TypeCacheHash should exist, otherwise this
+ * callback wouldn't be registered
+ */
+ if (OidIsValid(relid))
{
- if (typentry->typtype == TYPTYPE_COMPOSITE)
+ RelIdToTypeIdCacheEntry *relentry;
+
+ /*
+ * Find an RelIdToTypeIdCacheHash entry, which should exist as soon as
+ * corresponding typcache entry has something to clean.
+ */
+ relentry = (RelIdToTypeIdCacheEntry *) hash_search(RelIdToTypeIdCacheHash,
+ &relid,
+ HASH_FIND, NULL);
+
+ if (relentry != NULL)
{
- /* Skip if no match, unless we're zapping all composite types */
- if (relid != typentry->typrelid && relid != InvalidOid)
- continue;
+ typentry = (TypeCacheEntry *) hash_search(TypeCacheHash,
+ &relentry->composite_typid,
+ HASH_FIND, NULL);
- /* Delete tupdesc if we have it */
- if (typentry->tupDesc != NULL)
+ if (typentry != NULL)
{
- /*
- * Release our refcount, and free the tupdesc if none remain.
- * (Can't use DecrTupleDescRefCount because this reference is
- * not logged in current resource owner.)
- */
- Assert(typentry->tupDesc->tdrefcount > 0);
- if (--typentry->tupDesc->tdrefcount == 0)
- FreeTupleDesc(typentry->tupDesc);
- typentry->tupDesc = NULL;
+ Assert(typentry->typtype == TYPTYPE_COMPOSITE);
+ Assert(relid == typentry->typrelid);
- /*
- * Also clear tupDesc_identifier, so that anything watching
- * that will realize that the tupdesc has possibly changed.
- * (Alternatively, we could specify that to detect possible
- * tupdesc change, one must check for tupDesc != NULL as well
- * as tupDesc_identifier being the same as what was previously
- * seen. That seems error-prone.)
- */
- typentry->tupDesc_identifier = 0;
+ InvalidateCompositeTypeCacheEntry(typentry);
}
-
- /* Reset equality/comparison/hashing validity information */
- typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
}
- else if (typentry->typtype == TYPTYPE_DOMAIN)
+
+ /*
+ * Visit all the domain types sequentially. Typically this shouldn't
+ * affect performance since domain types are less tended to bloat.
+ * Domain types are created manually, unlike composite types which are
+ * automatically created for every temporary table.
+ */
+ for (typentry = firstDomainTypeEntry;
+ typentry != NULL;
+ typentry = typentry->nextDomain)
{
/*
* If it's domain over composite, reset flags. (We don't bother
@@ -2367,6 +2464,36 @@ TypeCacheRelCallback(Datum arg, Oid relid)
typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
}
}
+ else
+ {
+ HASH_SEQ_STATUS status;
+
+ /*
+ * Relid is invalid. By convention we need to reset all composite
+ * types in cache. Also, we should reset flags for domain types, and
+ * we loop over all entries in hash, so, do it in a single scan.
+ */
+ hash_seq_init(&status, TypeCacheHash);
+ while ((typentry = (TypeCacheEntry *) hash_seq_search(&status)) != NULL)
+ {
+ if (typentry->typtype == TYPTYPE_COMPOSITE)
+ {
+ InvalidateCompositeTypeCacheEntry(typentry);
+ }
+ else if (typentry->typtype == TYPTYPE_DOMAIN)
+ {
+ /*
+ * If it's domain over composite, reset flags. (We don't
+ * bother trying to determine whether the specific base type
+ * needs a reset.) Note that if we haven't determined whether
+ * the base type is composite, we don't need to reset
+ * anything.
+ */
+ if (typentry->flags & TCFLAGS_DOMAIN_BASE_IS_COMPOSITE)
+ typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
+ }
+ }
+ }
}
/*
@@ -2397,6 +2524,8 @@ TypeCacheTypCallback(Datum arg, int cacheid, uint32 hashvalue)
while ((typentry = (TypeCacheEntry *) hash_seq_search(&status)) != NULL)
{
+ bool hadPgTypeData = (typentry->flags & TCFLAGS_HAVE_PG_TYPE_DATA);
+
Assert(hashvalue == 0 || typentry->type_id_hash == hashvalue);
/*
@@ -2406,6 +2535,13 @@ TypeCacheTypCallback(Datum arg, int cacheid, uint32 hashvalue)
*/
typentry->flags &= ~(TCFLAGS_HAVE_PG_TYPE_DATA |
TCFLAGS_CHECKED_DOMAIN_CONSTRAINTS);
+
+ /*
+ * Call delete_rel_type_cache() if we cleaned
+ * TCFLAGS_HAVE_PG_TYPE_DATA flag previously.
+ */
+ if (hadPgTypeData)
+ delete_rel_type_cache_if_needed(typentry);
}
}
@@ -2914,3 +3050,129 @@ shared_record_typmod_registry_detach(dsm_segment *segment, Datum datum)
}
CurrentSession->shared_typmod_registry = NULL;
}
+
+/*
+ * Insert RelIdToTypeIdCacheHash entry if needed.
+ */
+static void
+insert_rel_type_cache_if_needed(TypeCacheEntry *typentry)
+{
+ /* Immediately quit for non-composite types */
+ if (typentry->typtype != TYPTYPE_COMPOSITE)
+ return;
+
+ /* typrelid should be given for composite types */
+ Assert(OidIsValid(typentry->typrelid));
+
+ /*
+ * Insert a RelIdToTypeIdCacheHash entry if the typentry have any
+ * information indicating it should be here.
+ */
+ if ((typentry->flags & TCFLAGS_HAVE_PG_TYPE_DATA) ||
+ (typentry->flags & TCFLAGS_OPERATOR_FLAGS) ||
+ typentry->tupDesc != NULL)
+ {
+ RelIdToTypeIdCacheEntry *relentry;
+ bool found;
+
+ relentry = (RelIdToTypeIdCacheEntry *) hash_search(RelIdToTypeIdCacheHash,
+ &typentry->typrelid,
+ HASH_ENTER, &found);
+ relentry->relid = typentry->typrelid;
+ relentry->composite_typid = typentry->type_id;
+ }
+}
+
+/*
+ * Delete entry RelIdToTypeIdCacheHash if needed after resetting of the
+ * TCFLAGS_HAVE_PG_TYPE_DATA flag, or any of TCFLAGS_OPERATOR_FLAGS flags,
+ * or tupDesc.
+ */
+static void
+delete_rel_type_cache_if_needed(TypeCacheEntry *typentry)
+{
+#ifdef USE_ASSERT_CHECKING
+ int i;
+ bool is_in_progress = false;
+
+ for (i = 0; i < in_progress_list_len; i++)
+ {
+ if (in_progress_list[i] == typentry->type_id)
+ {
+ is_in_progress = true;
+ break;
+ }
+ }
+#endif
+
+ /* Immediately quit for non-composite types */
+ if (typentry->typtype != TYPTYPE_COMPOSITE)
+ return;
+
+ /* typrelid should be given for composite types */
+ Assert(OidIsValid(typentry->typrelid));
+
+ /*
+ * Delete a RelIdToTypeIdCacheHash entry if the typentry doesn't have any
+ * information indicating entry should be still there.
+ */
+ if (!(typentry->flags & TCFLAGS_HAVE_PG_TYPE_DATA) &&
+ !(typentry->flags & TCFLAGS_OPERATOR_FLAGS) &&
+ typentry->tupDesc == NULL)
+ {
+ bool found;
+
+ (void) hash_search(RelIdToTypeIdCacheHash,
+ &typentry->typrelid,
+ HASH_REMOVE, &found);
+ Assert(found || is_in_progress);
+ }
+ else
+ {
+#ifdef USE_ASSERT_CHECKING
+ /*
+ * In assert-enabled builds otherwise check for RelIdToTypeIdCacheHash
+ * entry if it should exist.
+ */
+ bool found;
+
+ if (!is_in_progress)
+ {
+ (void) hash_search(RelIdToTypeIdCacheHash,
+ &typentry->typrelid,
+ HASH_FIND, &found);
+ Assert(found);
+ }
+#endif
+ }
+}
+
+static void
+cleanup_in_progress_typentries(void)
+{
+ int i;
+
+ for (i = 0; i < in_progress_list_len; i++)
+ {
+ TypeCacheEntry *typentry;
+
+ typentry = (TypeCacheEntry *) hash_search(TypeCacheHash,
+ &in_progress_list[i],
+ HASH_FIND, NULL);
+ insert_rel_type_cache_if_needed(typentry);
+ }
+
+ in_progress_list_len = 0;
+}
+
+void
+AtEOXact_TypeCache(void)
+{
+ cleanup_in_progress_typentries();
+}
+
+void
+AtEOSubXact_TypeCache(void)
+{
+ cleanup_in_progress_typentries();
+}
diff --git a/src/include/utils/typcache.h b/src/include/utils/typcache.h
index f506cc4aa35..f3d73ecee3a 100644
--- a/src/include/utils/typcache.h
+++ b/src/include/utils/typcache.h
@@ -207,4 +207,8 @@ extern void SharedRecordTypmodRegistryInit(SharedRecordTypmodRegistry *,
extern void SharedRecordTypmodRegistryAttach(SharedRecordTypmodRegistry *);
+extern void AtEOXact_TypeCache(void);
+
+extern void AtEOSubXact_TypeCache(void);
+
#endif /* TYPCACHE_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 57de1acff3a..9b3e7fd104b 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2378,6 +2378,7 @@ RelFileLocator
RelFileLocatorBackend
RelFileNumber
RelIdCacheEnt
+RelIdToTypeIdCacheEntry
RelInfo
RelInfoArr
RelMapFile
--
2.39.5 (Apple Git-154)
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2024-10-15 08:45 Artur Zakirov <[email protected]>
parent: Alexander Korotkov <[email protected]>
2 siblings, 0 replies; 55+ messages in thread
From: Artur Zakirov @ 2024-10-15 08:45 UTC (permalink / raw)
To: Alexander Korotkov <[email protected]>; +Cc: jian he <[email protected]>; Andrei Lepikhov <[email protected]>; Alexander Lakhin <[email protected]>; Pavel Borisov <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
On Tue, 15 Oct 2024 at 10:09, Alexander Korotkov <[email protected]> wrote:
> > /* Call check_delete_rel_type_cache() if we actually cleared something */
> > if (hadTupDescOrOpclass)
> > delete_rel_type_cache_if_needed(typentry);
> >
> > /*
> > * Call check_delete_rel_type_cache() if we cleaned
> > * TCFLAGS_HAVE_PG_TYPE_DATA flag previously.
> > */
> > if (hadPgTypeData)
> > delete_rel_type_cache_if_needed(typentry);
> >
> > check_delete_rel_type_cache don't exist, so these comments are wrong?
>
> Yep, they didn't get updated. Fixed in the attached patchset.
Thank you Alexander for the fixes. The last version of the patch looks
good to me.
> I'm not sure I get the point. This check ensures that type entry has
> something to be cleared. In this case we need to keep
> RelIdToTypeIdCacheHash entry to find this item on invalidation
> message. I'm not sure how TCFLAGS_DOMAIN_BASE_IS_COMPOSITE is
> relevant here, because it's valid only for TYPTYPE_DOMAIN while this
> patch deals with TYPTYPE_COMPOSITE.
Regarding this discussion earlier, I assumed that TYPTYPE_DOMAIN also
needs to be handled by `insert_rel_type_cache_if_needed()`. And it
seems that handling of TYPTYPE_DOMAIN will remain the same as before.
--
Kind regards,
Artur
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2024-10-15 09:50 jian he <[email protected]>
parent: Alexander Korotkov <[email protected]>
2 siblings, 2 replies; 55+ messages in thread
From: jian he @ 2024-10-15 09:50 UTC (permalink / raw)
To: Alexander Korotkov <[email protected]>; +Cc: Artur Zakirov <[email protected]>; Andrei Lepikhov <[email protected]>; Alexander Lakhin <[email protected]>; Pavel Borisov <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
On Tue, Oct 15, 2024 at 4:09 PM Alexander Korotkov <[email protected]> wrote:
>
> Hi, Jian!
>
> Thank you for your review.
>
> On Tue, Oct 15, 2024 at 10:34 AM jian he <[email protected]> wrote:
> > I don't fully understand all of it. but I did some tests anyway.
> >
> > static void
> > cleanup_in_progress_typentries(void)
> > {
> > int i;
> > if (in_progress_list_len > 1)
> > elog(INFO, "%s:%d in_progress_list_len > 1", __FILE_NAME__, __LINE__);
> > for (i = 0; i < in_progress_list_len; i++)
> > {
> > TypeCacheEntry *typentry;
> > typentry = (TypeCacheEntry *) hash_search(TypeCacheHash,
> > &in_progress_list[i],
> > HASH_FIND, NULL);
> > insert_rel_type_cache_if_needed(typentry);
> > }
> > in_progress_list_len = 0;
> > }
> >
> > the regress still passed.
> > I assume "elog(INFO, " won't interfere in cleanup_in_progress_typentries.
> > So we lack tests for larger in_progress_list_len values or i missed something?
>
> Try to run test suite with -DCLOBBER_CACHE_ALWAYS.
>
build from source, DCLOBBER_CACHE_ALWAYS takes a very long time.
So I gave up.
in lookup_type_cache, we unconditionally do
in_progress_list_len++;
in_progress_list_len--;
"static int in_progress_list_len;"
means in_progress_list_len value change is confined in
src/backend/utils/cache/typcache.c.
based on above information, i am still confused with
cleanup_in_progress_typentries, in_progress_list_len
is there any simple sql example to demo
cleanup_in_progress_typentries, in_progress_list_len> 0.
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2024-10-15 13:16 Artur Zakirov <[email protected]>
parent: jian he <[email protected]>
1 sibling, 0 replies; 55+ messages in thread
From: Artur Zakirov @ 2024-10-15 13:16 UTC (permalink / raw)
To: jian he <[email protected]>; +Cc: Alexander Korotkov <[email protected]>; Andrei Lepikhov <[email protected]>; Alexander Lakhin <[email protected]>; Pavel Borisov <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
On Tue, 15 Oct 2024 at 11:50, jian he <[email protected]> wrote:
> based on above information, i am still confused with
> cleanup_in_progress_typentries, in_progress_list_len
> is there any simple sql example to demo
> cleanup_in_progress_typentries, in_progress_list_len> 0.
AFAIK to reproduce cases when `in_progress_list_len > 0`
`lookup_type_cache()` should fail during its execution.
To do so you can call `lookup_type_cache()` with non-existing type_id
from a C function.
--
Kind regards,
Artur
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2024-10-17 09:41 Andrei Lepikhov <[email protected]>
parent: Alexander Korotkov <[email protected]>
2 siblings, 1 reply; 55+ messages in thread
From: Andrei Lepikhov @ 2024-10-17 09:41 UTC (permalink / raw)
To: Alexander Korotkov <[email protected]>; +Cc: Artur Zakirov <[email protected]>; Alexander Lakhin <[email protected]>; jian he <[email protected]>; Pavel Borisov <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
On 10/15/24 15:08, Alexander Korotkov wrote:
> Yep, they didn't get updated. Fixed in the attached patchset.
Let me wear Alexander Lakhin's mask for a moment and say that the code
may cause a segfault:
#0 0x000055e0da186000 in insert_rel_type_cache_if_needed (typentry=0x0)
at typcache.c:3066
b3066 if (typentry->typtype != TYPTYPE_COMPOSITE)
(gdb) bt 20
#0 0x000055e0da186000 in insert_rel_type_cache_if_needed (typentry=0x0)
at typcache.c:3066
#1 0x000055e0da18844f in cleanup_in_progress_typentries () at
typcache.c:3172
#2 0x000055e0da1883f9 in AtEOXact_TypeCache () at typcache.c:3181
#3 0x000055e0d9a22e59 in AbortTransaction () at xact.c:2961
#4 0x000055e0d9a1f75c in AbortCurrentTransactionInternal () at xact.c:3491
#5 0x000055e0d9a1f6be in AbortCurrentTransaction () at xact.c:3445
#6 0x000055e0d9f55f28 in PostgresMain (dbname=0x55e1057fb838
"regression", username=0x55e1057fb818 "danolivo") at postgres.c:4508
#7 0x000055e0d9f4e4a3 in BackendMain (startup_data=0x7ffeaf051310 "",
startup_data_len=4) at backend_startup.c:107
#8 0x000055e0d9e4bfee in postmaster_child_launch (child_type=B_BACKEND,
startup_data=0x7ffeaf051310 "", startup_data_len=4,
client_sock=0x7ffeaf051358) at launch_backend.c:274
#9 0x000055e0d9e522a3 in BackendStartup (client_sock=0x7ffeaf051358) at
postmaster.c:3420
#10 0x000055e0d9e502f9 in ServerLoop () at postmaster.c:1653
#11 0x000055e0d9e4f4fe in PostmasterMain (argc=3, argv=0x55e1057bb520)
at postmaster.c:1351
#12 0x000055e0d9cf4b2d in main (argc=3, argv=0x55e1057bb520) at main.c:197
It can happen if something triggers an error in the middle of
lookup_type_cache when in_progress_list[i] is already filled, but the
typentry wasn't created.
I think it can be easily shielded (see attached). Also, the name
cleanup_in_progress_typentries causes a lot of ponderings, I guess it
would be better to rename it likewise finalize_in_progress_typentries.
Also, I added trivial comments to better understand what the function does.
I think the first patch may already be committed, and this little burden
may be avoided in future versions.
--
regards, Andrei Lepikhov
Attachments:
[text/x-patch] minor-fix.diff (1.2K, ../../[email protected]/2-minor-fix.diff)
download | inline diff:
diff --git a/src/backend/utils/cache/typcache.c b/src/backend/utils/cache/typcache.c
index f54e7d531a..45aed74019 100644
--- a/src/backend/utils/cache/typcache.c
+++ b/src/backend/utils/cache/typcache.c
@@ -3058,7 +3058,7 @@ static void
insert_rel_type_cache_if_needed(TypeCacheEntry *typentry)
{
/* Immediately quit for non-composite types */
- if (typentry->typtype != TYPTYPE_COMPOSITE)
+ if (!typentry || typentry->typtype != TYPTYPE_COMPOSITE)
return;
/* typrelid should be given for composite types */
@@ -3147,8 +3147,13 @@ delete_rel_type_cache_if_needed(TypeCacheEntry *typentry)
}
}
+/*
+ * Add into the accessory hash table entries, added into TypCache and not added
+ * into the RelIdToTypeId matching hash table.
+ * It may happen in case of an error raised during the lookup_type_cache call.
+ */
static void
-cleanup_in_progress_typentries(void)
+finalize_in_progress_typentries(void)
{
int i;
@@ -3168,11 +3173,11 @@ cleanup_in_progress_typentries(void)
void
AtEOXact_TypeCache(void)
{
- cleanup_in_progress_typentries();
+ finalize_in_progress_typentries();
}
void
AtEOSubXact_TypeCache(void)
{
- cleanup_in_progress_typentries();
+ finalize_in_progress_typentries();
}
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2024-10-20 17:36 Alexander Korotkov <[email protected]>
parent: Andrei Lepikhov <[email protected]>
0 siblings, 1 reply; 55+ messages in thread
From: Alexander Korotkov @ 2024-10-20 17:36 UTC (permalink / raw)
To: Andrei Lepikhov <[email protected]>; +Cc: Artur Zakirov <[email protected]>; Alexander Lakhin <[email protected]>; jian he <[email protected]>; Pavel Borisov <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
On Thu, Oct 17, 2024 at 12:41 PM Andrei Lepikhov <[email protected]> wrote:
> On 10/15/24 15:08, Alexander Korotkov wrote:
> > Yep, they didn't get updated. Fixed in the attached patchset.
> Let me wear Alexander Lakhin's mask for a moment and say that the code
> may cause a segfault:
>
> #0 0x000055e0da186000 in insert_rel_type_cache_if_needed (typentry=0x0)
> at typcache.c:3066
> b3066 if (typentry->typtype != TYPTYPE_COMPOSITE)
> (gdb) bt 20
> #0 0x000055e0da186000 in insert_rel_type_cache_if_needed (typentry=0x0)
> at typcache.c:3066
> #1 0x000055e0da18844f in cleanup_in_progress_typentries () at
> typcache.c:3172
> #2 0x000055e0da1883f9 in AtEOXact_TypeCache () at typcache.c:3181
> #3 0x000055e0d9a22e59 in AbortTransaction () at xact.c:2961
> #4 0x000055e0d9a1f75c in AbortCurrentTransactionInternal () at xact.c:3491
> #5 0x000055e0d9a1f6be in AbortCurrentTransaction () at xact.c:3445
> #6 0x000055e0d9f55f28 in PostgresMain (dbname=0x55e1057fb838
> "regression", username=0x55e1057fb818 "danolivo") at postgres.c:4508
> #7 0x000055e0d9f4e4a3 in BackendMain (startup_data=0x7ffeaf051310 "",
> startup_data_len=4) at backend_startup.c:107
> #8 0x000055e0d9e4bfee in postmaster_child_launch (child_type=B_BACKEND,
> startup_data=0x7ffeaf051310 "", startup_data_len=4,
> client_sock=0x7ffeaf051358) at launch_backend.c:274
> #9 0x000055e0d9e522a3 in BackendStartup (client_sock=0x7ffeaf051358) at
> postmaster.c:3420
> #10 0x000055e0d9e502f9 in ServerLoop () at postmaster.c:1653
> #11 0x000055e0d9e4f4fe in PostmasterMain (argc=3, argv=0x55e1057bb520)
> at postmaster.c:1351
> #12 0x000055e0d9cf4b2d in main (argc=3, argv=0x55e1057bb520) at main.c:197
>
> It can happen if something triggers an error in the middle of
> lookup_type_cache when in_progress_list[i] is already filled, but the
> typentry wasn't created.
> I think it can be easily shielded (see attached). Also, the name
> cleanup_in_progress_typentries causes a lot of ponderings, I guess it
> would be better to rename it likewise finalize_in_progress_typentries.
> Also, I added trivial comments to better understand what the function does.
>
> I think the first patch may already be committed, and this little burden
> may be avoided in future versions.
Thank you!
I've integrated your patch. But I think
finalize_in_progress_typentries() is more appropriate place to check
typentry for NULL. Also, I've revised the
finalize_in_progress_typentries() header comment. I'm going to push
these patches if no objections.
------
Regards,
Alexander Korotkov
Supabase
Attachments:
[application/octet-stream] v14-0002-Avoid-looping-over-all-type-cache-entries-in-Typ.patch (18.3K, ../../CAPpHfdtj5yXgPiPJBenTLaFvBbF7U+LVpN=Eho1=o01jU_KQaA@mail.gmail.com/2-v14-0002-Avoid-looping-over-all-type-cache-entries-in-Typ.patch)
download | inline diff:
From a4bbadce7df1777eea8b98fb72f1c9163ac8ce11 Mon Sep 17 00:00:00 2001
From: Alexander Korotkov <[email protected]>
Date: Tue, 10 Sep 2024 23:25:04 +0300
Subject: [PATCH v14 2/2] Avoid looping over all type cache entries in
TypeCacheRelCallback()
Currently when a single relcache entry gets invalidated,
TypeCacheRelCallback() has to loop over all type cache entries to find
appropriate typentry to invalidate. Unfortunately, using the syscache here
is impossible, because this callback could be called outside a transaction
and this makes impossible catalog lookups. This is why present commit
introduces RelIdToTypeIdCacheHash to map relation OID to its composite type
OID.
We are keeping RelIdToTypeIdCacheHash entry while corresponding type cache
entry have something to clean. Therefore, RelIdToTypeIdCacheHash shouldn't
get bloat in the case of temporary tables flood.
Discussion: https://postgr.es/m/5812a6e5-68ae-4d84-9d85-b443176966a1%40sigaev.ru
Author: Teodor Sigaev
Reviewed-by: Aleksander Alekseev, Tom Lane, Michael Paquier, Roman Zharkov
Reviewed-by: Andrei Lepikhov, Pavel Borisov
---
src/backend/access/transam/xact.c | 10 +
src/backend/utils/cache/typcache.c | 356 +++++++++++++++++++++++++----
src/include/utils/typcache.h | 4 +
src/tools/pgindent/typedefs.list | 1 +
4 files changed, 327 insertions(+), 44 deletions(-)
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 87700c7c5c7..b0b05e28790 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -70,6 +70,7 @@
#include "utils/snapmgr.h"
#include "utils/timeout.h"
#include "utils/timestamp.h"
+#include "utils/typcache.h"
/*
* User-tweakable parameters
@@ -2407,6 +2408,9 @@ CommitTransaction(void)
/* Clean up the relation cache */
AtEOXact_RelationCache(true);
+ /* Clean up the type cache */
+ AtEOXact_TypeCache();
+
/*
* Make catalog changes visible to all backends. This has to happen after
* relcache references are dropped (see comments for
@@ -2709,6 +2713,9 @@ PrepareTransaction(void)
/* Clean up the relation cache */
AtEOXact_RelationCache(true);
+ /* Clean up the type cache */
+ AtEOXact_TypeCache();
+
/* notify doesn't need a postprepare call */
PostPrepare_PgStat();
@@ -2951,6 +2958,7 @@ AbortTransaction(void)
false, true);
AtEOXact_Buffers(false);
AtEOXact_RelationCache(false);
+ AtEOXact_TypeCache();
AtEOXact_Inval(false);
AtEOXact_MultiXact();
ResourceOwnerRelease(TopTransactionResourceOwner,
@@ -5153,6 +5161,7 @@ CommitSubTransaction(void)
true, false);
AtEOSubXact_RelationCache(true, s->subTransactionId,
s->parent->subTransactionId);
+ AtEOSubXact_TypeCache();
AtEOSubXact_Inval(true);
AtSubCommit_smgr();
@@ -5328,6 +5337,7 @@ AbortSubTransaction(void)
AtEOSubXact_RelationCache(false, s->subTransactionId,
s->parent->subTransactionId);
+ AtEOSubXact_TypeCache();
AtEOSubXact_Inval(false);
ResourceOwnerRelease(s->curTransactionOwner,
RESOURCE_RELEASE_LOCKS,
diff --git a/src/backend/utils/cache/typcache.c b/src/backend/utils/cache/typcache.c
index 11382547ec0..5df965443ae 100644
--- a/src/backend/utils/cache/typcache.c
+++ b/src/backend/utils/cache/typcache.c
@@ -77,6 +77,20 @@
/* The main type cache hashtable searched by lookup_type_cache */
static HTAB *TypeCacheHash = NULL;
+/*
+ * The mapping of relation's OID to the corresponding composite type OID.
+ * We're keeping the map entry when the corresponding typentry has something
+ * to clear i.e it has either TCFLAGS_HAVE_PG_TYPE_DATA, or
+ * TCFLAGS_OPERATOR_FLAGS, or tupdesc.
+ */
+static HTAB *RelIdToTypeIdCacheHash = NULL;
+
+typedef struct RelIdToTypeIdCacheEntry
+{
+ Oid relid; /* OID of the relation */
+ Oid composite_typid; /* OID of the relation's composite type */
+} RelIdToTypeIdCacheEntry;
+
/* List of type cache entries for domain types */
static TypeCacheEntry *firstDomainTypeEntry = NULL;
@@ -208,6 +222,10 @@ typedef struct SharedTypmodTableEntry
dsa_pointer shared_tupdesc;
} SharedTypmodTableEntry;
+static Oid *in_progress_list;
+static int in_progress_list_len;
+static int in_progress_list_maxlen;
+
/*
* A comparator function for SharedRecordTableKey.
*/
@@ -329,6 +347,8 @@ static void shared_record_typmod_registry_detach(dsm_segment *segment,
static TupleDesc find_or_make_matching_shared_tupledesc(TupleDesc tupdesc);
static dsa_pointer share_tupledesc(dsa_area *area, TupleDesc tupdesc,
uint32 typmod);
+static void insert_rel_type_cache_if_needed(TypeCacheEntry *typentry);
+static void delete_rel_type_cache_if_needed(TypeCacheEntry *typentry);
/*
@@ -366,11 +386,13 @@ lookup_type_cache(Oid type_id, int flags)
{
TypeCacheEntry *typentry;
bool found;
+ int in_progress_offset;
if (TypeCacheHash == NULL)
{
/* First time through: initialize the hash table */
HASHCTL ctl;
+ int allocsize;
ctl.keysize = sizeof(Oid);
ctl.entrysize = sizeof(TypeCacheEntry);
@@ -385,6 +407,13 @@ lookup_type_cache(Oid type_id, int flags)
TypeCacheHash = hash_create("Type information cache", 64,
&ctl, HASH_ELEM | HASH_FUNCTION);
+ Assert(RelIdToTypeIdCacheHash == NULL);
+
+ ctl.keysize = sizeof(Oid);
+ ctl.entrysize = sizeof(RelIdToTypeIdCacheEntry);
+ RelIdToTypeIdCacheHash = hash_create("Map from relid to OID of cached composite type", 64,
+ &ctl, HASH_ELEM | HASH_BLOBS);
+
/* Also set up callbacks for SI invalidations */
CacheRegisterRelcacheCallback(TypeCacheRelCallback, (Datum) 0);
CacheRegisterSyscacheCallback(TYPEOID, TypeCacheTypCallback, (Datum) 0);
@@ -394,8 +423,32 @@ lookup_type_cache(Oid type_id, int flags)
/* Also make sure CacheMemoryContext exists */
if (!CacheMemoryContext)
CreateCacheMemoryContext();
+
+ /*
+ * reserve enough in_progress_list slots for many cases
+ */
+ allocsize = 4;
+ in_progress_list =
+ MemoryContextAlloc(CacheMemoryContext,
+ allocsize * sizeof(*in_progress_list));
+ in_progress_list_maxlen = allocsize;
}
+ Assert(TypeCacheHash != NULL && RelIdToTypeIdCacheHash != NULL);
+
+ /* Register to catch invalidation messages */
+ if (in_progress_list_len >= in_progress_list_maxlen)
+ {
+ int allocsize;
+
+ allocsize = in_progress_list_maxlen * 2;
+ in_progress_list = repalloc(in_progress_list,
+ allocsize * sizeof(*in_progress_list));
+ in_progress_list_maxlen = allocsize;
+ }
+ in_progress_offset = in_progress_list_len++;
+ in_progress_list[in_progress_offset] = type_id;
+
/* Try to look up an existing entry */
typentry = (TypeCacheEntry *) hash_search(TypeCacheHash,
&type_id,
@@ -896,6 +949,11 @@ lookup_type_cache(Oid type_id, int flags)
load_domaintype_info(typentry);
}
+ Assert(in_progress_offset + 1 == in_progress_list_len);
+ in_progress_list_len--;
+
+ insert_rel_type_cache_if_needed(typentry);
+
return typentry;
}
@@ -2290,6 +2348,53 @@ SharedRecordTypmodRegistryAttach(SharedRecordTypmodRegistry *registry)
CurrentSession->shared_typmod_table = typmod_table;
}
+/*
+ * InvalidateCompositeTypeCacheEntry
+ * Invalidate particular TypeCacheEntry on Relcache inval callback
+ *
+ * Delete the cached tuple descriptor (if any) for the given composite
+ * type, and reset whatever info we have cached about the composite type's
+ * comparability.
+ */
+static void
+InvalidateCompositeTypeCacheEntry(TypeCacheEntry *typentry)
+{
+ bool hadTupDescOrOpclass;
+
+ Assert(typentry->typtype == TYPTYPE_COMPOSITE &&
+ OidIsValid(typentry->typrelid));
+
+ hadTupDescOrOpclass = (typentry->tupDesc != NULL) ||
+ (typentry->flags & TCFLAGS_OPERATOR_FLAGS);
+
+ /* Delete tupdesc if we have it */
+ if (typentry->tupDesc != NULL)
+ {
+ /*
+ * Release our refcount and free the tupdesc if none remain. We can't
+ * use DecrTupleDescRefCount here because this reference is not logged
+ * by the current resource owner.
+ */
+ Assert(typentry->tupDesc->tdrefcount > 0);
+ if (--typentry->tupDesc->tdrefcount == 0)
+ FreeTupleDesc(typentry->tupDesc);
+ typentry->tupDesc = NULL;
+
+ /*
+ * Also clear tupDesc_identifier, so that anyone watching it will
+ * realize that the tupdesc has changed.
+ */
+ typentry->tupDesc_identifier = 0;
+ }
+
+ /* Reset equality/comparison/hashing validity information */
+ typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
+
+ /* Call delete_rel_type_cache() if we actually cleared something */
+ if (hadTupDescOrOpclass)
+ delete_rel_type_cache_if_needed(typentry);
+}
+
/*
* TypeCacheRelCallback
* Relcache inval callback function
@@ -2299,63 +2404,55 @@ SharedRecordTypmodRegistryAttach(SharedRecordTypmodRegistry *registry)
* whatever info we have cached about the composite type's comparability.
*
* This is called when a relcache invalidation event occurs for the given
- * relid. We must scan the whole typcache hash since we don't know the
- * type OID corresponding to the relid. We could do a direct search if this
- * were a syscache-flush callback on pg_type, but then we would need all
- * ALTER-TABLE-like commands that could modify a rowtype to issue syscache
- * invals against the rel's pg_type OID. The extra SI signaling could very
- * well cost more than we'd save, since in most usages there are not very
- * many entries in a backend's typcache. The risk of bugs-of-omission seems
- * high, too.
- *
- * Another possibility, with only localized impact, is to maintain a second
- * hashtable that indexes composite-type typcache entries by their typrelid.
- * But it's still not clear it's worth the trouble.
+ * relid. We can't use syscache to find a type corresponding to the given
+ * relation because the code can be called outside of transaction. Thus we use
+ * the RelIdToTypeIdCacheHash map to locate appropriate typcache entry.
*/
static void
TypeCacheRelCallback(Datum arg, Oid relid)
{
- HASH_SEQ_STATUS status;
TypeCacheEntry *typentry;
- /* TypeCacheHash must exist, else this callback wouldn't be registered */
- hash_seq_init(&status, TypeCacheHash);
- while ((typentry = (TypeCacheEntry *) hash_seq_search(&status)) != NULL)
+ /*
+ * RelIdToTypeIdCacheHash and TypeCacheHash should exist, otherwise this
+ * callback wouldn't be registered
+ */
+ if (OidIsValid(relid))
{
- if (typentry->typtype == TYPTYPE_COMPOSITE)
+ RelIdToTypeIdCacheEntry *relentry;
+
+ /*
+ * Find an RelIdToTypeIdCacheHash entry, which should exist as soon as
+ * corresponding typcache entry has something to clean.
+ */
+ relentry = (RelIdToTypeIdCacheEntry *) hash_search(RelIdToTypeIdCacheHash,
+ &relid,
+ HASH_FIND, NULL);
+
+ if (relentry != NULL)
{
- /* Skip if no match, unless we're zapping all composite types */
- if (relid != typentry->typrelid && relid != InvalidOid)
- continue;
+ typentry = (TypeCacheEntry *) hash_search(TypeCacheHash,
+ &relentry->composite_typid,
+ HASH_FIND, NULL);
- /* Delete tupdesc if we have it */
- if (typentry->tupDesc != NULL)
+ if (typentry != NULL)
{
- /*
- * Release our refcount, and free the tupdesc if none remain.
- * (Can't use DecrTupleDescRefCount because this reference is
- * not logged in current resource owner.)
- */
- Assert(typentry->tupDesc->tdrefcount > 0);
- if (--typentry->tupDesc->tdrefcount == 0)
- FreeTupleDesc(typentry->tupDesc);
- typentry->tupDesc = NULL;
+ Assert(typentry->typtype == TYPTYPE_COMPOSITE);
+ Assert(relid == typentry->typrelid);
- /*
- * Also clear tupDesc_identifier, so that anything watching
- * that will realize that the tupdesc has possibly changed.
- * (Alternatively, we could specify that to detect possible
- * tupdesc change, one must check for tupDesc != NULL as well
- * as tupDesc_identifier being the same as what was previously
- * seen. That seems error-prone.)
- */
- typentry->tupDesc_identifier = 0;
+ InvalidateCompositeTypeCacheEntry(typentry);
}
-
- /* Reset equality/comparison/hashing validity information */
- typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
}
- else if (typentry->typtype == TYPTYPE_DOMAIN)
+
+ /*
+ * Visit all the domain types sequentially. Typically this shouldn't
+ * affect performance since domain types are less tended to bloat.
+ * Domain types are created manually, unlike composite types which are
+ * automatically created for every temporary table.
+ */
+ for (typentry = firstDomainTypeEntry;
+ typentry != NULL;
+ typentry = typentry->nextDomain)
{
/*
* If it's domain over composite, reset flags. (We don't bother
@@ -2367,6 +2464,36 @@ TypeCacheRelCallback(Datum arg, Oid relid)
typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
}
}
+ else
+ {
+ HASH_SEQ_STATUS status;
+
+ /*
+ * Relid is invalid. By convention we need to reset all composite
+ * types in cache. Also, we should reset flags for domain types, and
+ * we loop over all entries in hash, so, do it in a single scan.
+ */
+ hash_seq_init(&status, TypeCacheHash);
+ while ((typentry = (TypeCacheEntry *) hash_seq_search(&status)) != NULL)
+ {
+ if (typentry->typtype == TYPTYPE_COMPOSITE)
+ {
+ InvalidateCompositeTypeCacheEntry(typentry);
+ }
+ else if (typentry->typtype == TYPTYPE_DOMAIN)
+ {
+ /*
+ * If it's domain over composite, reset flags. (We don't
+ * bother trying to determine whether the specific base type
+ * needs a reset.) Note that if we haven't determined whether
+ * the base type is composite, we don't need to reset
+ * anything.
+ */
+ if (typentry->flags & TCFLAGS_DOMAIN_BASE_IS_COMPOSITE)
+ typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
+ }
+ }
+ }
}
/*
@@ -2397,6 +2524,8 @@ TypeCacheTypCallback(Datum arg, int cacheid, uint32 hashvalue)
while ((typentry = (TypeCacheEntry *) hash_seq_search(&status)) != NULL)
{
+ bool hadPgTypeData = (typentry->flags & TCFLAGS_HAVE_PG_TYPE_DATA);
+
Assert(hashvalue == 0 || typentry->type_id_hash == hashvalue);
/*
@@ -2406,6 +2535,13 @@ TypeCacheTypCallback(Datum arg, int cacheid, uint32 hashvalue)
*/
typentry->flags &= ~(TCFLAGS_HAVE_PG_TYPE_DATA |
TCFLAGS_CHECKED_DOMAIN_CONSTRAINTS);
+
+ /*
+ * Call delete_rel_type_cache() if we cleaned
+ * TCFLAGS_HAVE_PG_TYPE_DATA flag previously.
+ */
+ if (hadPgTypeData)
+ delete_rel_type_cache_if_needed(typentry);
}
}
@@ -2914,3 +3050,135 @@ shared_record_typmod_registry_detach(dsm_segment *segment, Datum datum)
}
CurrentSession->shared_typmod_registry = NULL;
}
+
+/*
+ * Insert RelIdToTypeIdCacheHash entry if needed.
+ */
+static void
+insert_rel_type_cache_if_needed(TypeCacheEntry *typentry)
+{
+ /* Immediately quit for non-composite types */
+ if (typentry->typtype != TYPTYPE_COMPOSITE)
+ return;
+
+ /* typrelid should be given for composite types */
+ Assert(OidIsValid(typentry->typrelid));
+
+ /*
+ * Insert a RelIdToTypeIdCacheHash entry if the typentry have any
+ * information indicating it should be here.
+ */
+ if ((typentry->flags & TCFLAGS_HAVE_PG_TYPE_DATA) ||
+ (typentry->flags & TCFLAGS_OPERATOR_FLAGS) ||
+ typentry->tupDesc != NULL)
+ {
+ RelIdToTypeIdCacheEntry *relentry;
+ bool found;
+
+ relentry = (RelIdToTypeIdCacheEntry *) hash_search(RelIdToTypeIdCacheHash,
+ &typentry->typrelid,
+ HASH_ENTER, &found);
+ relentry->relid = typentry->typrelid;
+ relentry->composite_typid = typentry->type_id;
+ }
+}
+
+/*
+ * Delete entry RelIdToTypeIdCacheHash if needed after resetting of the
+ * TCFLAGS_HAVE_PG_TYPE_DATA flag, or any of TCFLAGS_OPERATOR_FLAGS flags,
+ * or tupDesc.
+ */
+static void
+delete_rel_type_cache_if_needed(TypeCacheEntry *typentry)
+{
+#ifdef USE_ASSERT_CHECKING
+ int i;
+ bool is_in_progress = false;
+
+ for (i = 0; i < in_progress_list_len; i++)
+ {
+ if (in_progress_list[i] == typentry->type_id)
+ {
+ is_in_progress = true;
+ break;
+ }
+ }
+#endif
+
+ /* Immediately quit for non-composite types */
+ if (typentry->typtype != TYPTYPE_COMPOSITE)
+ return;
+
+ /* typrelid should be given for composite types */
+ Assert(OidIsValid(typentry->typrelid));
+
+ /*
+ * Delete a RelIdToTypeIdCacheHash entry if the typentry doesn't have any
+ * information indicating entry should be still there.
+ */
+ if (!(typentry->flags & TCFLAGS_HAVE_PG_TYPE_DATA) &&
+ !(typentry->flags & TCFLAGS_OPERATOR_FLAGS) &&
+ typentry->tupDesc == NULL)
+ {
+ bool found;
+
+ (void) hash_search(RelIdToTypeIdCacheHash,
+ &typentry->typrelid,
+ HASH_REMOVE, &found);
+ Assert(found || is_in_progress);
+ }
+ else
+ {
+#ifdef USE_ASSERT_CHECKING
+ /*
+ * In assert-enabled builds otherwise check for RelIdToTypeIdCacheHash
+ * entry if it should exist.
+ */
+ bool found;
+
+ if (!is_in_progress)
+ {
+ (void) hash_search(RelIdToTypeIdCacheHash,
+ &typentry->typrelid,
+ HASH_FIND, &found);
+ Assert(found);
+ }
+#endif
+ }
+}
+
+/*
+ * Add possibly missing RelIdToTypeId entries related to TypeCacheHas
+ * entries, marked as in-progress by lookup_type_cache(). It may happen
+ * in case of an error or interruption during the lookup_type_cache() call.
+ */
+static void
+finalize_in_progress_typentries(void)
+{
+ int i;
+
+ for (i = 0; i < in_progress_list_len; i++)
+ {
+ TypeCacheEntry *typentry;
+
+ typentry = (TypeCacheEntry *) hash_search(TypeCacheHash,
+ &in_progress_list[i],
+ HASH_FIND, NULL);
+ if (typentry)
+ insert_rel_type_cache_if_needed(typentry);
+ }
+
+ in_progress_list_len = 0;
+}
+
+void
+AtEOXact_TypeCache(void)
+{
+ finalize_in_progress_typentries();
+}
+
+void
+AtEOSubXact_TypeCache(void)
+{
+ finalize_in_progress_typentries();
+}
diff --git a/src/include/utils/typcache.h b/src/include/utils/typcache.h
index f506cc4aa35..f3d73ecee3a 100644
--- a/src/include/utils/typcache.h
+++ b/src/include/utils/typcache.h
@@ -207,4 +207,8 @@ extern void SharedRecordTypmodRegistryInit(SharedRecordTypmodRegistry *,
extern void SharedRecordTypmodRegistryAttach(SharedRecordTypmodRegistry *);
+extern void AtEOXact_TypeCache(void);
+
+extern void AtEOSubXact_TypeCache(void);
+
#endif /* TYPCACHE_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 57de1acff3a..9b3e7fd104b 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2378,6 +2378,7 @@ RelFileLocator
RelFileLocatorBackend
RelFileNumber
RelIdCacheEnt
+RelIdToTypeIdCacheEntry
RelInfo
RelInfoArr
RelMapFile
--
2.39.5 (Apple Git-154)
[application/octet-stream] v14-0001-Update-header-comment-for-lookup_type_cache.patch (1.4K, ../../CAPpHfdtj5yXgPiPJBenTLaFvBbF7U+LVpN=Eho1=o01jU_KQaA@mail.gmail.com/3-v14-0001-Update-header-comment-for-lookup_type_cache.patch)
download | inline diff:
From d624b14c9a4bd93426a5e475d4503589927ef2ae Mon Sep 17 00:00:00 2001
From: Alexander Korotkov <[email protected]>
Date: Fri, 13 Sep 2024 02:10:04 +0300
Subject: [PATCH v14 1/2] Update header comment for lookup_type_cache()
Describe the way we handle concurrent invalidation messages.
---
src/backend/utils/cache/typcache.c | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/src/backend/utils/cache/typcache.c b/src/backend/utils/cache/typcache.c
index 2ec136b7d30..11382547ec0 100644
--- a/src/backend/utils/cache/typcache.c
+++ b/src/backend/utils/cache/typcache.c
@@ -351,6 +351,15 @@ type_cache_syshash(const void *key, Size keysize)
* invalid. Note however that we may fail to find one or more of the
* values requested by 'flags'; the caller needs to check whether the fields
* are InvalidOid or not.
+ *
+ * Note that while filling TypeCacheEntry we might process concurrent
+ * invalidation messages, causing our not-yet-filled TypeCacheEntry to be
+ * invalidated. In this case, we typically only clear flags while values are
+ * still available for the caller. It's expected that the caller holds
+ * enough locks on type-depending objects that the values are still relevant.
+ * It's also important that the tupdesc is filled in the last for
+ * TYPTYPE_COMPOSITE. So, it can't get invalidated during the
+ * lookup_type_cache() call.
*/
TypeCacheEntry *
lookup_type_cache(Oid type_id, int flags)
--
2.39.5 (Apple Git-154)
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2024-10-20 17:47 Alexander Korotkov <[email protected]>
parent: jian he <[email protected]>
1 sibling, 1 reply; 55+ messages in thread
From: Alexander Korotkov @ 2024-10-20 17:47 UTC (permalink / raw)
To: jian he <[email protected]>; +Cc: Artur Zakirov <[email protected]>; Andrei Lepikhov <[email protected]>; Alexander Lakhin <[email protected]>; Pavel Borisov <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
On Tue, Oct 15, 2024 at 12:50 PM jian he <[email protected]> wrote:
>
> On Tue, Oct 15, 2024 at 4:09 PM Alexander Korotkov <[email protected]> wrote:
> >
> > Hi, Jian!
> >
> > Thank you for your review.
> >
> > On Tue, Oct 15, 2024 at 10:34 AM jian he <[email protected]> wrote:
> > > I don't fully understand all of it. but I did some tests anyway.
> > >
> > > static void
> > > cleanup_in_progress_typentries(void)
> > > {
> > > int i;
> > > if (in_progress_list_len > 1)
> > > elog(INFO, "%s:%d in_progress_list_len > 1", __FILE_NAME__, __LINE__);
> > > for (i = 0; i < in_progress_list_len; i++)
> > > {
> > > TypeCacheEntry *typentry;
> > > typentry = (TypeCacheEntry *) hash_search(TypeCacheHash,
> > > &in_progress_list[i],
> > > HASH_FIND, NULL);
> > > insert_rel_type_cache_if_needed(typentry);
> > > }
> > > in_progress_list_len = 0;
> > > }
> > >
> > > the regress still passed.
> > > I assume "elog(INFO, " won't interfere in cleanup_in_progress_typentries.
> > > So we lack tests for larger in_progress_list_len values or i missed something?
> >
> > Try to run test suite with -DCLOBBER_CACHE_ALWAYS.
> >
>
> build from source, DCLOBBER_CACHE_ALWAYS takes a very long time.
> So I gave up.
>
>
> in lookup_type_cache, we unconditionally do
> in_progress_list_len++;
> in_progress_list_len--;
Yes, this should work OK when no errors. On error or interruption,
finalize_in_progress_typentries() will clean the things up.
> "static int in_progress_list_len;"
> means in_progress_list_len value change is confined in
> src/backend/utils/cache/typcache.c.
Yep.
> based on above information, i am still confused with
> cleanup_in_progress_typentries, in_progress_list_len
> is there any simple sql example to demo
> cleanup_in_progress_typentries, in_progress_list_len> 0.
I don't think there is simple sql to reliably reproduce that. In
order to hit that, we must process invalidation messages in some
(short) moment of time during lookup_type_cache(). You can reproduce
that by setting a breakpoint in lookup_type_cache() and in parallel do
something to invalidate the type cache entry (for instance, ALTER
TABLE ... ADD COLUMN ... would invalidate the composite type). In
principle, we can reproduce that using injection points. However, I'm
not intended to do that as long as we have buildfarm members with
-DCLOBBER_CACHE_ALWAYS. FWIW, I will for sure run tests with
-DCLOBBER_CACHE_ALWAYS before committing this.
------
Regards,
Alexander Korotkov
Supabase
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2024-10-20 18:00 Alexander Korotkov <[email protected]>
parent: Alexander Korotkov <[email protected]>
0 siblings, 1 reply; 55+ messages in thread
From: Alexander Korotkov @ 2024-10-20 18:00 UTC (permalink / raw)
To: jian he <[email protected]>; +Cc: Artur Zakirov <[email protected]>; Andrei Lepikhov <[email protected]>; Alexander Lakhin <[email protected]>; Pavel Borisov <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
On Sun, Oct 20, 2024 at 8:47 PM Alexander Korotkov <[email protected]> wrote:
> On Tue, Oct 15, 2024 at 12:50 PM jian he <[email protected]> wrote:
> > build from source, DCLOBBER_CACHE_ALWAYS takes a very long time.
> > So I gave up.
> >
> >
> > in lookup_type_cache, we unconditionally do
> > in_progress_list_len++;
> > in_progress_list_len--;
>
> Yes, this should work OK when no errors. On error or interruption,
> finalize_in_progress_typentries() will clean the things up.
>
> > "static int in_progress_list_len;"
> > means in_progress_list_len value change is confined in
> > src/backend/utils/cache/typcache.c.
>
> Yep.
>
> > based on above information, i am still confused with
> > cleanup_in_progress_typentries, in_progress_list_len
> > is there any simple sql example to demo
> > cleanup_in_progress_typentries, in_progress_list_len> 0.
>
> I don't think there is simple sql to reliably reproduce that. In
> order to hit that, we must process invalidation messages in some
> (short) moment of time during lookup_type_cache(). You can reproduce
> that by setting a breakpoint in lookup_type_cache() and in parallel do
> something to invalidate the type cache entry (for instance, ALTER
> TABLE ... ADD COLUMN ... would invalidate the composite type).
Oops, concurrent invalidation message is not enough here. So,
-DCLOBBER_CACHE_ALWAYS is also not enough to reproduce the situation.
Injection-point test is required. I'm going to add this.
------
Regards,
Alexander Korotkov
Supabase
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2024-10-20 22:09 Alexander Korotkov <[email protected]>
parent: Alexander Korotkov <[email protected]>
0 siblings, 2 replies; 55+ messages in thread
From: Alexander Korotkov @ 2024-10-20 22:09 UTC (permalink / raw)
To: jian he <[email protected]>; +Cc: Artur Zakirov <[email protected]>; Andrei Lepikhov <[email protected]>; Alexander Lakhin <[email protected]>; Pavel Borisov <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
On Sun, Oct 20, 2024 at 9:00 PM Alexander Korotkov <[email protected]> wrote:
> On Sun, Oct 20, 2024 at 8:47 PM Alexander Korotkov <[email protected]> wrote:
> > On Tue, Oct 15, 2024 at 12:50 PM jian he <[email protected]> wrote:
> > > build from source, DCLOBBER_CACHE_ALWAYS takes a very long time.
> > > So I gave up.
> > >
> > >
> > > in lookup_type_cache, we unconditionally do
> > > in_progress_list_len++;
> > > in_progress_list_len--;
> >
> > Yes, this should work OK when no errors. On error or interruption,
> > finalize_in_progress_typentries() will clean the things up.
> >
> > > "static int in_progress_list_len;"
> > > means in_progress_list_len value change is confined in
> > > src/backend/utils/cache/typcache.c.
> >
> > Yep.
> >
> > > based on above information, i am still confused with
> > > cleanup_in_progress_typentries, in_progress_list_len
> > > is there any simple sql example to demo
> > > cleanup_in_progress_typentries, in_progress_list_len> 0.
> >
> > I don't think there is simple sql to reliably reproduce that. In
> > order to hit that, we must process invalidation messages in some
> > (short) moment of time during lookup_type_cache(). You can reproduce
> > that by setting a breakpoint in lookup_type_cache() and in parallel do
> > something to invalidate the type cache entry (for instance, ALTER
> > TABLE ... ADD COLUMN ... would invalidate the composite type).
>
> Oops, concurrent invalidation message is not enough here. So,
> -DCLOBBER_CACHE_ALWAYS is also not enough to reproduce the situation.
> Injection-point test is required. I'm going to add this.
Here you go. The test with injection point is implemented.
------
Regards,
Alexander Korotkov
Supabase
Attachments:
[application/octet-stream] v15-0001-Update-header-comment-for-lookup_type_cache.patch (1.4K, ../../CAPpHfdvK8XS-FvXSxcEr-=azS3SHJo3b0yrQCq4uOv4fsw0JGw@mail.gmail.com/2-v15-0001-Update-header-comment-for-lookup_type_cache.patch)
download | inline diff:
From d624b14c9a4bd93426a5e475d4503589927ef2ae Mon Sep 17 00:00:00 2001
From: Alexander Korotkov <[email protected]>
Date: Fri, 13 Sep 2024 02:10:04 +0300
Subject: [PATCH v15 1/2] Update header comment for lookup_type_cache()
Describe the way we handle concurrent invalidation messages.
---
src/backend/utils/cache/typcache.c | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/src/backend/utils/cache/typcache.c b/src/backend/utils/cache/typcache.c
index 2ec136b7d30..11382547ec0 100644
--- a/src/backend/utils/cache/typcache.c
+++ b/src/backend/utils/cache/typcache.c
@@ -351,6 +351,15 @@ type_cache_syshash(const void *key, Size keysize)
* invalid. Note however that we may fail to find one or more of the
* values requested by 'flags'; the caller needs to check whether the fields
* are InvalidOid or not.
+ *
+ * Note that while filling TypeCacheEntry we might process concurrent
+ * invalidation messages, causing our not-yet-filled TypeCacheEntry to be
+ * invalidated. In this case, we typically only clear flags while values are
+ * still available for the caller. It's expected that the caller holds
+ * enough locks on type-depending objects that the values are still relevant.
+ * It's also important that the tupdesc is filled in the last for
+ * TYPTYPE_COMPOSITE. So, it can't get invalidated during the
+ * lookup_type_cache() call.
*/
TypeCacheEntry *
lookup_type_cache(Oid type_id, int flags)
--
2.39.5 (Apple Git-154)
[application/octet-stream] v15-0002-Avoid-looping-over-all-type-cache-entries-in-Typ.patch (24.7K, ../../CAPpHfdvK8XS-FvXSxcEr-=azS3SHJo3b0yrQCq4uOv4fsw0JGw@mail.gmail.com/3-v15-0002-Avoid-looping-over-all-type-cache-entries-in-Typ.patch)
download | inline diff:
From 9412092fc39d2989167bb2d48606e0c27a755a5b Mon Sep 17 00:00:00 2001
From: Alexander Korotkov <[email protected]>
Date: Tue, 10 Sep 2024 23:25:04 +0300
Subject: [PATCH v15 2/2] Avoid looping over all type cache entries in
TypeCacheRelCallback()
Currently when a single relcache entry gets invalidated,
TypeCacheRelCallback() has to loop over all type cache entries to find
appropriate typentry to invalidate. Unfortunately, using the syscache here
is impossible, because this callback could be called outside a transaction
and this makes impossible catalog lookups. This is why present commit
introduces RelIdToTypeIdCacheHash to map relation OID to its composite type
OID.
We are keeping RelIdToTypeIdCacheHash entry while corresponding type cache
entry have something to clean. Therefore, RelIdToTypeIdCacheHash shouldn't
get bloat in the case of temporary tables flood.
There are many places in lookup_type_cache() where syscache invalidation,
user interruption, or even error could occur. In order to handle this, we
keep an array of in-progress type cache entries. In the case of
lookup_type_cache() interruption this array is processed to keep
RelIdToTypeIdCacheHash in a consistent state.
Discussion: https://postgr.es/m/5812a6e5-68ae-4d84-9d85-b443176966a1%40sigaev.ru
Author: Teodor Sigaev
Reviewed-by: Aleksander Alekseev, Tom Lane, Michael Paquier, Roman Zharkov
Reviewed-by: Andrei Lepikhov, Pavel Borisov
---
src/backend/access/transam/xact.c | 10 +
src/backend/utils/cache/typcache.c | 359 +++++++++++++++---
src/include/utils/typcache.h | 4 +
src/test/modules/Makefile | 4 +-
src/test/modules/meson.build | 1 +
src/test/modules/typcache/.gitignore | 4 +
src/test/modules/typcache/Makefile | 28 ++
.../expected/typcache_rel_type_cache.out | 34 ++
src/test/modules/typcache/meson.build | 16 +
.../typcache/sql/typcache_rel_type_cache.sql | 18 +
src/tools/pgindent/typedefs.list | 1 +
11 files changed, 433 insertions(+), 46 deletions(-)
create mode 100644 src/test/modules/typcache/.gitignore
create mode 100644 src/test/modules/typcache/Makefile
create mode 100644 src/test/modules/typcache/expected/typcache_rel_type_cache.out
create mode 100644 src/test/modules/typcache/meson.build
create mode 100644 src/test/modules/typcache/sql/typcache_rel_type_cache.sql
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 87700c7c5c7..b0b05e28790 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -70,6 +70,7 @@
#include "utils/snapmgr.h"
#include "utils/timeout.h"
#include "utils/timestamp.h"
+#include "utils/typcache.h"
/*
* User-tweakable parameters
@@ -2407,6 +2408,9 @@ CommitTransaction(void)
/* Clean up the relation cache */
AtEOXact_RelationCache(true);
+ /* Clean up the type cache */
+ AtEOXact_TypeCache();
+
/*
* Make catalog changes visible to all backends. This has to happen after
* relcache references are dropped (see comments for
@@ -2709,6 +2713,9 @@ PrepareTransaction(void)
/* Clean up the relation cache */
AtEOXact_RelationCache(true);
+ /* Clean up the type cache */
+ AtEOXact_TypeCache();
+
/* notify doesn't need a postprepare call */
PostPrepare_PgStat();
@@ -2951,6 +2958,7 @@ AbortTransaction(void)
false, true);
AtEOXact_Buffers(false);
AtEOXact_RelationCache(false);
+ AtEOXact_TypeCache();
AtEOXact_Inval(false);
AtEOXact_MultiXact();
ResourceOwnerRelease(TopTransactionResourceOwner,
@@ -5153,6 +5161,7 @@ CommitSubTransaction(void)
true, false);
AtEOSubXact_RelationCache(true, s->subTransactionId,
s->parent->subTransactionId);
+ AtEOSubXact_TypeCache();
AtEOSubXact_Inval(true);
AtSubCommit_smgr();
@@ -5328,6 +5337,7 @@ AbortSubTransaction(void)
AtEOSubXact_RelationCache(false, s->subTransactionId,
s->parent->subTransactionId);
+ AtEOSubXact_TypeCache();
AtEOSubXact_Inval(false);
ResourceOwnerRelease(s->curTransactionOwner,
RESOURCE_RELEASE_LOCKS,
diff --git a/src/backend/utils/cache/typcache.c b/src/backend/utils/cache/typcache.c
index 11382547ec0..6fa525d08f0 100644
--- a/src/backend/utils/cache/typcache.c
+++ b/src/backend/utils/cache/typcache.c
@@ -66,6 +66,7 @@
#include "utils/builtins.h"
#include "utils/catcache.h"
#include "utils/fmgroids.h"
+#include "utils/injection_point.h"
#include "utils/inval.h"
#include "utils/lsyscache.h"
#include "utils/memutils.h"
@@ -77,6 +78,20 @@
/* The main type cache hashtable searched by lookup_type_cache */
static HTAB *TypeCacheHash = NULL;
+/*
+ * The mapping of relation's OID to the corresponding composite type OID.
+ * We're keeping the map entry when the corresponding typentry has something
+ * to clear i.e it has either TCFLAGS_HAVE_PG_TYPE_DATA, or
+ * TCFLAGS_OPERATOR_FLAGS, or tupdesc.
+ */
+static HTAB *RelIdToTypeIdCacheHash = NULL;
+
+typedef struct RelIdToTypeIdCacheEntry
+{
+ Oid relid; /* OID of the relation */
+ Oid composite_typid; /* OID of the relation's composite type */
+} RelIdToTypeIdCacheEntry;
+
/* List of type cache entries for domain types */
static TypeCacheEntry *firstDomainTypeEntry = NULL;
@@ -208,6 +223,10 @@ typedef struct SharedTypmodTableEntry
dsa_pointer shared_tupdesc;
} SharedTypmodTableEntry;
+static Oid *in_progress_list;
+static int in_progress_list_len;
+static int in_progress_list_maxlen;
+
/*
* A comparator function for SharedRecordTableKey.
*/
@@ -329,6 +348,8 @@ static void shared_record_typmod_registry_detach(dsm_segment *segment,
static TupleDesc find_or_make_matching_shared_tupledesc(TupleDesc tupdesc);
static dsa_pointer share_tupledesc(dsa_area *area, TupleDesc tupdesc,
uint32 typmod);
+static void insert_rel_type_cache_if_needed(TypeCacheEntry *typentry);
+static void delete_rel_type_cache_if_needed(TypeCacheEntry *typentry);
/*
@@ -366,11 +387,13 @@ lookup_type_cache(Oid type_id, int flags)
{
TypeCacheEntry *typentry;
bool found;
+ int in_progress_offset;
if (TypeCacheHash == NULL)
{
/* First time through: initialize the hash table */
HASHCTL ctl;
+ int allocsize;
ctl.keysize = sizeof(Oid);
ctl.entrysize = sizeof(TypeCacheEntry);
@@ -385,6 +408,13 @@ lookup_type_cache(Oid type_id, int flags)
TypeCacheHash = hash_create("Type information cache", 64,
&ctl, HASH_ELEM | HASH_FUNCTION);
+ Assert(RelIdToTypeIdCacheHash == NULL);
+
+ ctl.keysize = sizeof(Oid);
+ ctl.entrysize = sizeof(RelIdToTypeIdCacheEntry);
+ RelIdToTypeIdCacheHash = hash_create("Map from relid to OID of cached composite type", 64,
+ &ctl, HASH_ELEM | HASH_BLOBS);
+
/* Also set up callbacks for SI invalidations */
CacheRegisterRelcacheCallback(TypeCacheRelCallback, (Datum) 0);
CacheRegisterSyscacheCallback(TYPEOID, TypeCacheTypCallback, (Datum) 0);
@@ -394,8 +424,32 @@ lookup_type_cache(Oid type_id, int flags)
/* Also make sure CacheMemoryContext exists */
if (!CacheMemoryContext)
CreateCacheMemoryContext();
+
+ /*
+ * reserve enough in_progress_list slots for many cases
+ */
+ allocsize = 4;
+ in_progress_list =
+ MemoryContextAlloc(CacheMemoryContext,
+ allocsize * sizeof(*in_progress_list));
+ in_progress_list_maxlen = allocsize;
}
+ Assert(TypeCacheHash != NULL && RelIdToTypeIdCacheHash != NULL);
+
+ /* Register to catch invalidation messages */
+ if (in_progress_list_len >= in_progress_list_maxlen)
+ {
+ int allocsize;
+
+ allocsize = in_progress_list_maxlen * 2;
+ in_progress_list = repalloc(in_progress_list,
+ allocsize * sizeof(*in_progress_list));
+ in_progress_list_maxlen = allocsize;
+ }
+ in_progress_offset = in_progress_list_len++;
+ in_progress_list[in_progress_offset] = type_id;
+
/* Try to look up an existing entry */
typentry = (TypeCacheEntry *) hash_search(TypeCacheHash,
&type_id,
@@ -896,6 +950,13 @@ lookup_type_cache(Oid type_id, int flags)
load_domaintype_info(typentry);
}
+ INJECTION_POINT("typecache-before-rel-type-cache-insert");
+
+ Assert(in_progress_offset + 1 == in_progress_list_len);
+ in_progress_list_len--;
+
+ insert_rel_type_cache_if_needed(typentry);
+
return typentry;
}
@@ -2290,6 +2351,53 @@ SharedRecordTypmodRegistryAttach(SharedRecordTypmodRegistry *registry)
CurrentSession->shared_typmod_table = typmod_table;
}
+/*
+ * InvalidateCompositeTypeCacheEntry
+ * Invalidate particular TypeCacheEntry on Relcache inval callback
+ *
+ * Delete the cached tuple descriptor (if any) for the given composite
+ * type, and reset whatever info we have cached about the composite type's
+ * comparability.
+ */
+static void
+InvalidateCompositeTypeCacheEntry(TypeCacheEntry *typentry)
+{
+ bool hadTupDescOrOpclass;
+
+ Assert(typentry->typtype == TYPTYPE_COMPOSITE &&
+ OidIsValid(typentry->typrelid));
+
+ hadTupDescOrOpclass = (typentry->tupDesc != NULL) ||
+ (typentry->flags & TCFLAGS_OPERATOR_FLAGS);
+
+ /* Delete tupdesc if we have it */
+ if (typentry->tupDesc != NULL)
+ {
+ /*
+ * Release our refcount and free the tupdesc if none remain. We can't
+ * use DecrTupleDescRefCount here because this reference is not logged
+ * by the current resource owner.
+ */
+ Assert(typentry->tupDesc->tdrefcount > 0);
+ if (--typentry->tupDesc->tdrefcount == 0)
+ FreeTupleDesc(typentry->tupDesc);
+ typentry->tupDesc = NULL;
+
+ /*
+ * Also clear tupDesc_identifier, so that anyone watching it will
+ * realize that the tupdesc has changed.
+ */
+ typentry->tupDesc_identifier = 0;
+ }
+
+ /* Reset equality/comparison/hashing validity information */
+ typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
+
+ /* Call delete_rel_type_cache() if we actually cleared something */
+ if (hadTupDescOrOpclass)
+ delete_rel_type_cache_if_needed(typentry);
+}
+
/*
* TypeCacheRelCallback
* Relcache inval callback function
@@ -2299,63 +2407,55 @@ SharedRecordTypmodRegistryAttach(SharedRecordTypmodRegistry *registry)
* whatever info we have cached about the composite type's comparability.
*
* This is called when a relcache invalidation event occurs for the given
- * relid. We must scan the whole typcache hash since we don't know the
- * type OID corresponding to the relid. We could do a direct search if this
- * were a syscache-flush callback on pg_type, but then we would need all
- * ALTER-TABLE-like commands that could modify a rowtype to issue syscache
- * invals against the rel's pg_type OID. The extra SI signaling could very
- * well cost more than we'd save, since in most usages there are not very
- * many entries in a backend's typcache. The risk of bugs-of-omission seems
- * high, too.
- *
- * Another possibility, with only localized impact, is to maintain a second
- * hashtable that indexes composite-type typcache entries by their typrelid.
- * But it's still not clear it's worth the trouble.
+ * relid. We can't use syscache to find a type corresponding to the given
+ * relation because the code can be called outside of transaction. Thus we use
+ * the RelIdToTypeIdCacheHash map to locate appropriate typcache entry.
*/
static void
TypeCacheRelCallback(Datum arg, Oid relid)
{
- HASH_SEQ_STATUS status;
TypeCacheEntry *typentry;
- /* TypeCacheHash must exist, else this callback wouldn't be registered */
- hash_seq_init(&status, TypeCacheHash);
- while ((typentry = (TypeCacheEntry *) hash_seq_search(&status)) != NULL)
+ /*
+ * RelIdToTypeIdCacheHash and TypeCacheHash should exist, otherwise this
+ * callback wouldn't be registered
+ */
+ if (OidIsValid(relid))
{
- if (typentry->typtype == TYPTYPE_COMPOSITE)
+ RelIdToTypeIdCacheEntry *relentry;
+
+ /*
+ * Find an RelIdToTypeIdCacheHash entry, which should exist as soon as
+ * corresponding typcache entry has something to clean.
+ */
+ relentry = (RelIdToTypeIdCacheEntry *) hash_search(RelIdToTypeIdCacheHash,
+ &relid,
+ HASH_FIND, NULL);
+
+ if (relentry != NULL)
{
- /* Skip if no match, unless we're zapping all composite types */
- if (relid != typentry->typrelid && relid != InvalidOid)
- continue;
+ typentry = (TypeCacheEntry *) hash_search(TypeCacheHash,
+ &relentry->composite_typid,
+ HASH_FIND, NULL);
- /* Delete tupdesc if we have it */
- if (typentry->tupDesc != NULL)
+ if (typentry != NULL)
{
- /*
- * Release our refcount, and free the tupdesc if none remain.
- * (Can't use DecrTupleDescRefCount because this reference is
- * not logged in current resource owner.)
- */
- Assert(typentry->tupDesc->tdrefcount > 0);
- if (--typentry->tupDesc->tdrefcount == 0)
- FreeTupleDesc(typentry->tupDesc);
- typentry->tupDesc = NULL;
+ Assert(typentry->typtype == TYPTYPE_COMPOSITE);
+ Assert(relid == typentry->typrelid);
- /*
- * Also clear tupDesc_identifier, so that anything watching
- * that will realize that the tupdesc has possibly changed.
- * (Alternatively, we could specify that to detect possible
- * tupdesc change, one must check for tupDesc != NULL as well
- * as tupDesc_identifier being the same as what was previously
- * seen. That seems error-prone.)
- */
- typentry->tupDesc_identifier = 0;
+ InvalidateCompositeTypeCacheEntry(typentry);
}
-
- /* Reset equality/comparison/hashing validity information */
- typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
}
- else if (typentry->typtype == TYPTYPE_DOMAIN)
+
+ /*
+ * Visit all the domain types sequentially. Typically this shouldn't
+ * affect performance since domain types are less tended to bloat.
+ * Domain types are created manually, unlike composite types which are
+ * automatically created for every temporary table.
+ */
+ for (typentry = firstDomainTypeEntry;
+ typentry != NULL;
+ typentry = typentry->nextDomain)
{
/*
* If it's domain over composite, reset flags. (We don't bother
@@ -2367,6 +2467,36 @@ TypeCacheRelCallback(Datum arg, Oid relid)
typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
}
}
+ else
+ {
+ HASH_SEQ_STATUS status;
+
+ /*
+ * Relid is invalid. By convention we need to reset all composite
+ * types in cache. Also, we should reset flags for domain types, and
+ * we loop over all entries in hash, so, do it in a single scan.
+ */
+ hash_seq_init(&status, TypeCacheHash);
+ while ((typentry = (TypeCacheEntry *) hash_seq_search(&status)) != NULL)
+ {
+ if (typentry->typtype == TYPTYPE_COMPOSITE)
+ {
+ InvalidateCompositeTypeCacheEntry(typentry);
+ }
+ else if (typentry->typtype == TYPTYPE_DOMAIN)
+ {
+ /*
+ * If it's domain over composite, reset flags. (We don't
+ * bother trying to determine whether the specific base type
+ * needs a reset.) Note that if we haven't determined whether
+ * the base type is composite, we don't need to reset
+ * anything.
+ */
+ if (typentry->flags & TCFLAGS_DOMAIN_BASE_IS_COMPOSITE)
+ typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
+ }
+ }
+ }
}
/*
@@ -2397,6 +2527,8 @@ TypeCacheTypCallback(Datum arg, int cacheid, uint32 hashvalue)
while ((typentry = (TypeCacheEntry *) hash_seq_search(&status)) != NULL)
{
+ bool hadPgTypeData = (typentry->flags & TCFLAGS_HAVE_PG_TYPE_DATA);
+
Assert(hashvalue == 0 || typentry->type_id_hash == hashvalue);
/*
@@ -2406,6 +2538,13 @@ TypeCacheTypCallback(Datum arg, int cacheid, uint32 hashvalue)
*/
typentry->flags &= ~(TCFLAGS_HAVE_PG_TYPE_DATA |
TCFLAGS_CHECKED_DOMAIN_CONSTRAINTS);
+
+ /*
+ * Call delete_rel_type_cache() if we cleaned
+ * TCFLAGS_HAVE_PG_TYPE_DATA flag previously.
+ */
+ if (hadPgTypeData)
+ delete_rel_type_cache_if_needed(typentry);
}
}
@@ -2914,3 +3053,135 @@ shared_record_typmod_registry_detach(dsm_segment *segment, Datum datum)
}
CurrentSession->shared_typmod_registry = NULL;
}
+
+/*
+ * Insert RelIdToTypeIdCacheHash entry if needed.
+ */
+static void
+insert_rel_type_cache_if_needed(TypeCacheEntry *typentry)
+{
+ /* Immediately quit for non-composite types */
+ if (typentry->typtype != TYPTYPE_COMPOSITE)
+ return;
+
+ /* typrelid should be given for composite types */
+ Assert(OidIsValid(typentry->typrelid));
+
+ /*
+ * Insert a RelIdToTypeIdCacheHash entry if the typentry have any
+ * information indicating it should be here.
+ */
+ if ((typentry->flags & TCFLAGS_HAVE_PG_TYPE_DATA) ||
+ (typentry->flags & TCFLAGS_OPERATOR_FLAGS) ||
+ typentry->tupDesc != NULL)
+ {
+ RelIdToTypeIdCacheEntry *relentry;
+ bool found;
+
+ relentry = (RelIdToTypeIdCacheEntry *) hash_search(RelIdToTypeIdCacheHash,
+ &typentry->typrelid,
+ HASH_ENTER, &found);
+ relentry->relid = typentry->typrelid;
+ relentry->composite_typid = typentry->type_id;
+ }
+}
+
+/*
+ * Delete entry RelIdToTypeIdCacheHash if needed after resetting of the
+ * TCFLAGS_HAVE_PG_TYPE_DATA flag, or any of TCFLAGS_OPERATOR_FLAGS flags,
+ * or tupDesc.
+ */
+static void
+delete_rel_type_cache_if_needed(TypeCacheEntry *typentry)
+{
+#ifdef USE_ASSERT_CHECKING
+ int i;
+ bool is_in_progress = false;
+
+ for (i = 0; i < in_progress_list_len; i++)
+ {
+ if (in_progress_list[i] == typentry->type_id)
+ {
+ is_in_progress = true;
+ break;
+ }
+ }
+#endif
+
+ /* Immediately quit for non-composite types */
+ if (typentry->typtype != TYPTYPE_COMPOSITE)
+ return;
+
+ /* typrelid should be given for composite types */
+ Assert(OidIsValid(typentry->typrelid));
+
+ /*
+ * Delete a RelIdToTypeIdCacheHash entry if the typentry doesn't have any
+ * information indicating entry should be still there.
+ */
+ if (!(typentry->flags & TCFLAGS_HAVE_PG_TYPE_DATA) &&
+ !(typentry->flags & TCFLAGS_OPERATOR_FLAGS) &&
+ typentry->tupDesc == NULL)
+ {
+ bool found;
+
+ (void) hash_search(RelIdToTypeIdCacheHash,
+ &typentry->typrelid,
+ HASH_REMOVE, &found);
+ Assert(found || is_in_progress);
+ }
+ else
+ {
+#ifdef USE_ASSERT_CHECKING
+ /*
+ * In assert-enabled builds otherwise check for RelIdToTypeIdCacheHash
+ * entry if it should exist.
+ */
+ bool found;
+
+ if (!is_in_progress)
+ {
+ (void) hash_search(RelIdToTypeIdCacheHash,
+ &typentry->typrelid,
+ HASH_FIND, &found);
+ Assert(found);
+ }
+#endif
+ }
+}
+
+/*
+ * Add possibly missing RelIdToTypeId entries related to TypeCacheHas
+ * entries, marked as in-progress by lookup_type_cache(). It may happen
+ * in case of an error or interruption during the lookup_type_cache() call.
+ */
+static void
+finalize_in_progress_typentries(void)
+{
+ int i;
+
+ for (i = 0; i < in_progress_list_len; i++)
+ {
+ TypeCacheEntry *typentry;
+
+ typentry = (TypeCacheEntry *) hash_search(TypeCacheHash,
+ &in_progress_list[i],
+ HASH_FIND, NULL);
+ if (typentry)
+ insert_rel_type_cache_if_needed(typentry);
+ }
+
+ in_progress_list_len = 0;
+}
+
+void
+AtEOXact_TypeCache(void)
+{
+ finalize_in_progress_typentries();
+}
+
+void
+AtEOSubXact_TypeCache(void)
+{
+ finalize_in_progress_typentries();
+}
diff --git a/src/include/utils/typcache.h b/src/include/utils/typcache.h
index f506cc4aa35..f3d73ecee3a 100644
--- a/src/include/utils/typcache.h
+++ b/src/include/utils/typcache.h
@@ -207,4 +207,8 @@ extern void SharedRecordTypmodRegistryInit(SharedRecordTypmodRegistry *,
extern void SharedRecordTypmodRegistryAttach(SharedRecordTypmodRegistry *);
+extern void AtEOXact_TypeCache(void);
+
+extern void AtEOSubXact_TypeCache(void);
+
#endif /* TYPCACHE_H */
diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile
index 256799f520a..c0d3cf0e14b 100644
--- a/src/test/modules/Makefile
+++ b/src/test/modules/Makefile
@@ -43,9 +43,9 @@ SUBDIRS = \
ifeq ($(enable_injection_points),yes)
-SUBDIRS += injection_points gin
+SUBDIRS += injection_points gin typcache
else
-ALWAYS_SUBDIRS += injection_points gin
+ALWAYS_SUBDIRS += injection_points gin typcache
endif
ifeq ($(with_ssl),openssl)
diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build
index d8fe059d236..c829b619530 100644
--- a/src/test/modules/meson.build
+++ b/src/test/modules/meson.build
@@ -36,6 +36,7 @@ subdir('test_rls_hooks')
subdir('test_shm_mq')
subdir('test_slru')
subdir('test_tidstore')
+subdir('typcache')
subdir('unsafe_tests')
subdir('worker_spi')
subdir('xid_wraparound')
diff --git a/src/test/modules/typcache/.gitignore b/src/test/modules/typcache/.gitignore
new file mode 100644
index 00000000000..5dcb3ff9723
--- /dev/null
+++ b/src/test/modules/typcache/.gitignore
@@ -0,0 +1,4 @@
+# Generated subdirectories
+/log/
+/results/
+/tmp_check/
diff --git a/src/test/modules/typcache/Makefile b/src/test/modules/typcache/Makefile
new file mode 100644
index 00000000000..6ee46ec0891
--- /dev/null
+++ b/src/test/modules/typcache/Makefile
@@ -0,0 +1,28 @@
+# src/test/modules/typcache/Makefile
+
+EXTRA_INSTALL = src/test/modules/typcache_rel_type_cache.out.out
+
+REGRESS = typcache_rel_type_cache
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = src/test/modules/typcache
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+
+# XXX: This test is conditional on enable_injection_points in the
+# parent Makefile, so we should never get here in the first place if
+# injection points are not enabled. But the buildfarm 'misc-check'
+# step doesn't pay attention to the if-condition in the parent
+# Makefile. To work around that, disable running the test here too.
+ifeq ($(enable_injection_points),yes)
+include $(top_srcdir)/contrib/contrib-global.mk
+else
+check:
+ @echo "injection points are disabled in this build"
+endif
+
+endif
diff --git a/src/test/modules/typcache/expected/typcache_rel_type_cache.out b/src/test/modules/typcache/expected/typcache_rel_type_cache.out
new file mode 100644
index 00000000000..b113e0bbd5d
--- /dev/null
+++ b/src/test/modules/typcache/expected/typcache_rel_type_cache.out
@@ -0,0 +1,34 @@
+--
+-- This test checks that lookup_type_cache() can correctly handle an
+-- interruption. We use the injection point to simulate an error but note
+-- that a similar situation could happen due to user query interruption.
+-- Despite the interruption, a map entry from the relation oid to type cache
+-- entry should be created. This is validated by subsequent modification of
+-- the table schema, then type casts which use new schema implying
+-- successful type cache invalidation by relation oid.
+--
+CREATE EXTENSION injection_points;
+CREATE TABLE t (i int);
+SELECT injection_points_attach('typecache-before-rel-type-cache-insert', 'error');
+ injection_points_attach
+-------------------------
+
+(1 row)
+
+SELECT '(1)'::t;
+ERROR: error triggered for injection point typecache-before-rel-type-cache-insert
+LINE 1: SELECT '(1)'::t;
+ ^
+SELECT injection_points_detach('typecache-before-rel-type-cache-insert');
+ injection_points_detach
+-------------------------
+
+(1 row)
+
+ALTER TABLE t ADD COLUMN j int;
+SELECT '(1,2)'::t;
+ t
+-------
+ (1,2)
+(1 row)
+
diff --git a/src/test/modules/typcache/meson.build b/src/test/modules/typcache/meson.build
new file mode 100644
index 00000000000..cb2e34c0d2b
--- /dev/null
+++ b/src/test/modules/typcache/meson.build
@@ -0,0 +1,16 @@
+# Copyright (c) 2022-2024, PostgreSQL Global Development Group
+
+if not get_option('injection_points')
+ subdir_done()
+endif
+
+tests += {
+ 'name': 'typcache',
+ 'sd': meson.current_source_dir(),
+ 'bd': meson.current_build_dir(),
+ 'regress': {
+ 'sql': [
+ 'typcache_rel_type_cache',
+ ],
+ },
+}
diff --git a/src/test/modules/typcache/sql/typcache_rel_type_cache.sql b/src/test/modules/typcache/sql/typcache_rel_type_cache.sql
new file mode 100644
index 00000000000..2c0a434d988
--- /dev/null
+++ b/src/test/modules/typcache/sql/typcache_rel_type_cache.sql
@@ -0,0 +1,18 @@
+--
+-- This test checks that lookup_type_cache() can correctly handle an
+-- interruption. We use the injection point to simulate an error but note
+-- that a similar situation could happen due to user query interruption.
+-- Despite the interruption, a map entry from the relation oid to type cache
+-- entry should be created. This is validated by subsequent modification of
+-- the table schema, then type casts which use new schema implying
+-- successful type cache invalidation by relation oid.
+--
+
+CREATE EXTENSION injection_points;
+
+CREATE TABLE t (i int);
+SELECT injection_points_attach('typecache-before-rel-type-cache-insert', 'error');
+SELECT '(1)'::t;
+SELECT injection_points_detach('typecache-before-rel-type-cache-insert');
+ALTER TABLE t ADD COLUMN j int;
+SELECT '(1,2)'::t;
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 57de1acff3a..9b3e7fd104b 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2378,6 +2378,7 @@ RelFileLocator
RelFileLocatorBackend
RelFileNumber
RelIdCacheEnt
+RelIdToTypeIdCacheEntry
RelInfo
RelInfoArr
RelMapFile
--
2.39.5 (Apple Git-154)
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2024-10-20 23:32 Dagfinn Ilmari Mannsåker <[email protected]>
parent: Alexander Korotkov <[email protected]>
1 sibling, 1 reply; 55+ messages in thread
From: Dagfinn Ilmari Mannsåker @ 2024-10-20 23:32 UTC (permalink / raw)
To: Alexander Korotkov <[email protected]>; +Cc: jian he <[email protected]>; Artur Zakirov <[email protected]>; Andrei Lepikhov <[email protected]>; Alexander Lakhin <[email protected]>; Pavel Borisov <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
Alexander Korotkov <[email protected]> writes:
> +static Oid *in_progress_list;
> +static int in_progress_list_len;
> +static int in_progress_list_maxlen;
Is there any particular reason not to use pg_list.h for this?
- ilmari
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2024-10-21 05:36 Andrei Lepikhov <[email protected]>
parent: Alexander Korotkov <[email protected]>
0 siblings, 0 replies; 55+ messages in thread
From: Andrei Lepikhov @ 2024-10-21 05:36 UTC (permalink / raw)
To: Alexander Korotkov <[email protected]>; +Cc: Artur Zakirov <[email protected]>; Alexander Lakhin <[email protected]>; jian he <[email protected]>; Pavel Borisov <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
On 21/10/2024 00:36, Alexander Korotkov wrote:
> On Thu, Oct 17, 2024 at 12:41 PM Andrei Lepikhov <[email protected]> wrote:
>> I think the first patch may already be committed, and this little burden
>> may be avoided in future versions.
> I've integrated your patch. But I think
> finalize_in_progress_typentries() is more appropriate place to check
> typentry for NULL. Also, I've revised the
> finalize_in_progress_typentries() header comment. I'm going to push
> these patches if no objections.
I agree with your idea. Also, I think it would be more conventional not
to check the type entry for a NULL value but to test the 'found' value
instead.
And thanks for the injection points tests! I must have slipped my mind
about this option.
Now, the patch set looks good for me.
--
regards, Andrei Lepikhov
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2024-10-21 05:40 Andrei Lepikhov <[email protected]>
parent: Dagfinn Ilmari Mannsåker <[email protected]>
0 siblings, 1 reply; 55+ messages in thread
From: Andrei Lepikhov @ 2024-10-21 05:40 UTC (permalink / raw)
To: Dagfinn Ilmari Mannsåker <[email protected]>; Alexander Korotkov <[email protected]>; +Cc: jian he <[email protected]>; Artur Zakirov <[email protected]>; Alexander Lakhin <[email protected]>; Pavel Borisov <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
On 21/10/2024 06:32, Dagfinn Ilmari Mannsåker wrote:
> Alexander Korotkov <[email protected]> writes:
>
>> +static Oid *in_progress_list;
>> +static int in_progress_list_len;
>> +static int in_progress_list_maxlen;
>
> Is there any particular reason not to use pg_list.h for this?
Sure. The type cache lookup has to be as much optimal as possible.
Using an array and relating sequential access to it, we avoid memory
allocations and deallocations 99.9% of the time. Also, quick access to
the single element (which we will have in real life almost all of the
time) is much faster than employing list machinery.
--
regards, Andrei Lepikhov
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2024-10-21 07:51 jian he <[email protected]>
parent: Alexander Korotkov <[email protected]>
1 sibling, 1 reply; 55+ messages in thread
From: jian he @ 2024-10-21 07:51 UTC (permalink / raw)
To: Alexander Korotkov <[email protected]>; +Cc: Artur Zakirov <[email protected]>; Andrei Lepikhov <[email protected]>; Alexander Lakhin <[email protected]>; Pavel Borisov <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
thanks for the
INJECTION_POINT("typecache-before-rel-type-cache-insert");
Now I have better understanding of the whole changes.
+/*
+ * Add possibly missing RelIdToTypeId entries related to TypeCacheHas
+ * entries, marked as in-progress by lookup_type_cache(). It may happen
+ * in case of an error or interruption during the lookup_type_cache() call.
+ */
+static void
+finalize_in_progress_typentries(void)
comment typo. "TypeCacheHas" should be "TypeCacheHash"
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2024-10-21 08:10 Alexander Korotkov <[email protected]>
parent: Andrei Lepikhov <[email protected]>
0 siblings, 1 reply; 55+ messages in thread
From: Alexander Korotkov @ 2024-10-21 08:10 UTC (permalink / raw)
To: Andrei Lepikhov <[email protected]>; +Cc: Dagfinn Ilmari Mannsåker <[email protected]>; jian he <[email protected]>; Artur Zakirov <[email protected]>; Alexander Lakhin <[email protected]>; Pavel Borisov <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
On Mon, Oct 21, 2024 at 8:40 AM Andrei Lepikhov <[email protected]> wrote:
>
> On 21/10/2024 06:32, Dagfinn Ilmari Mannsåker wrote:
> > Alexander Korotkov <[email protected]> writes:
> >
> >> +static Oid *in_progress_list;
> >> +static int in_progress_list_len;
> >> +static int in_progress_list_maxlen;
> >
> > Is there any particular reason not to use pg_list.h for this?
> Sure. The type cache lookup has to be as much optimal as possible.
> Using an array and relating sequential access to it, we avoid memory
> allocations and deallocations 99.9% of the time. Also, quick access to
> the single element (which we will have in real life almost all of the
> time) is much faster than employing list machinery.
+1,
List with zero elements has to be NIL. That means continuous
allocations/deallocations.
------
Regards,
Alexander Korotkov
Supabase
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2024-10-21 08:11 Alexander Korotkov <[email protected]>
parent: jian he <[email protected]>
0 siblings, 0 replies; 55+ messages in thread
From: Alexander Korotkov @ 2024-10-21 08:11 UTC (permalink / raw)
To: jian he <[email protected]>; +Cc: Artur Zakirov <[email protected]>; Andrei Lepikhov <[email protected]>; Alexander Lakhin <[email protected]>; Pavel Borisov <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
On Mon, Oct 21, 2024 at 10:51 AM jian he <[email protected]> wrote:
>
> thanks for the
> INJECTION_POINT("typecache-before-rel-type-cache-insert");
> Now I have better understanding of the whole changes.
>
>
> +/*
> + * Add possibly missing RelIdToTypeId entries related to TypeCacheHas
> + * entries, marked as in-progress by lookup_type_cache(). It may happen
> + * in case of an error or interruption during the lookup_type_cache() call.
> + */
> +static void
> +finalize_in_progress_typentries(void)
> comment typo. "TypeCacheHas" should be "TypeCacheHash"
Thank you. This also has been spotted by Alexander Lakhin (off-list).
Fixed in the attached revision of the patchset.
------
Regards,
Alexander Korotkov
Supabase
Attachments:
[application/octet-stream] v16-0001-Update-header-comment-for-lookup_type_cache.patch (1.4K, ../../CAPpHfdsk+KfUTS2bUu+=eEJjPjz=mY_manKCYVMS6khvn_MStA@mail.gmail.com/2-v16-0001-Update-header-comment-for-lookup_type_cache.patch)
download | inline diff:
From d624b14c9a4bd93426a5e475d4503589927ef2ae Mon Sep 17 00:00:00 2001
From: Alexander Korotkov <[email protected]>
Date: Fri, 13 Sep 2024 02:10:04 +0300
Subject: [PATCH v16 1/2] Update header comment for lookup_type_cache()
Describe the way we handle concurrent invalidation messages.
---
src/backend/utils/cache/typcache.c | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/src/backend/utils/cache/typcache.c b/src/backend/utils/cache/typcache.c
index 2ec136b7d30..11382547ec0 100644
--- a/src/backend/utils/cache/typcache.c
+++ b/src/backend/utils/cache/typcache.c
@@ -351,6 +351,15 @@ type_cache_syshash(const void *key, Size keysize)
* invalid. Note however that we may fail to find one or more of the
* values requested by 'flags'; the caller needs to check whether the fields
* are InvalidOid or not.
+ *
+ * Note that while filling TypeCacheEntry we might process concurrent
+ * invalidation messages, causing our not-yet-filled TypeCacheEntry to be
+ * invalidated. In this case, we typically only clear flags while values are
+ * still available for the caller. It's expected that the caller holds
+ * enough locks on type-depending objects that the values are still relevant.
+ * It's also important that the tupdesc is filled in the last for
+ * TYPTYPE_COMPOSITE. So, it can't get invalidated during the
+ * lookup_type_cache() call.
*/
TypeCacheEntry *
lookup_type_cache(Oid type_id, int flags)
--
2.39.5 (Apple Git-154)
[application/octet-stream] v16-0002-Avoid-looping-over-all-type-cache-entries-in-Typ.patch (24.7K, ../../CAPpHfdsk+KfUTS2bUu+=eEJjPjz=mY_manKCYVMS6khvn_MStA@mail.gmail.com/3-v16-0002-Avoid-looping-over-all-type-cache-entries-in-Typ.patch)
download | inline diff:
From e126267c3a6babbc5d37924947ba8f7ec9120cec Mon Sep 17 00:00:00 2001
From: Alexander Korotkov <[email protected]>
Date: Tue, 10 Sep 2024 23:25:04 +0300
Subject: [PATCH v16 2/2] Avoid looping over all type cache entries in
TypeCacheRelCallback()
Currently when a single relcache entry gets invalidated,
TypeCacheRelCallback() has to loop over all type cache entries to find
appropriate typentry to invalidate. Unfortunately, using the syscache here
is impossible, because this callback could be called outside a transaction
and this makes impossible catalog lookups. This is why present commit
introduces RelIdToTypeIdCacheHash to map relation OID to its composite type
OID.
We are keeping RelIdToTypeIdCacheHash entry while corresponding type cache
entry have something to clean. Therefore, RelIdToTypeIdCacheHash shouldn't
get bloat in the case of temporary tables flood.
There are many places in lookup_type_cache() where syscache invalidation,
user interruption, or even error could occur. In order to handle this, we
keep an array of in-progress type cache entries. In the case of
lookup_type_cache() interruption this array is processed to keep
RelIdToTypeIdCacheHash in a consistent state.
Discussion: https://postgr.es/m/5812a6e5-68ae-4d84-9d85-b443176966a1%40sigaev.ru
Author: Teodor Sigaev
Reviewed-by: Aleksander Alekseev, Tom Lane, Michael Paquier, Roman Zharkov
Reviewed-by: Andrei Lepikhov, Pavel Borisov, Jian He, Alexander Lakhin
---
src/backend/access/transam/xact.c | 10 +
src/backend/utils/cache/typcache.c | 359 +++++++++++++++---
src/include/utils/typcache.h | 4 +
src/test/modules/Makefile | 4 +-
src/test/modules/meson.build | 1 +
src/test/modules/typcache/.gitignore | 4 +
src/test/modules/typcache/Makefile | 28 ++
.../expected/typcache_rel_type_cache.out | 34 ++
src/test/modules/typcache/meson.build | 16 +
.../typcache/sql/typcache_rel_type_cache.sql | 18 +
src/tools/pgindent/typedefs.list | 1 +
11 files changed, 433 insertions(+), 46 deletions(-)
create mode 100644 src/test/modules/typcache/.gitignore
create mode 100644 src/test/modules/typcache/Makefile
create mode 100644 src/test/modules/typcache/expected/typcache_rel_type_cache.out
create mode 100644 src/test/modules/typcache/meson.build
create mode 100644 src/test/modules/typcache/sql/typcache_rel_type_cache.sql
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 87700c7c5c7..b0b05e28790 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -70,6 +70,7 @@
#include "utils/snapmgr.h"
#include "utils/timeout.h"
#include "utils/timestamp.h"
+#include "utils/typcache.h"
/*
* User-tweakable parameters
@@ -2407,6 +2408,9 @@ CommitTransaction(void)
/* Clean up the relation cache */
AtEOXact_RelationCache(true);
+ /* Clean up the type cache */
+ AtEOXact_TypeCache();
+
/*
* Make catalog changes visible to all backends. This has to happen after
* relcache references are dropped (see comments for
@@ -2709,6 +2713,9 @@ PrepareTransaction(void)
/* Clean up the relation cache */
AtEOXact_RelationCache(true);
+ /* Clean up the type cache */
+ AtEOXact_TypeCache();
+
/* notify doesn't need a postprepare call */
PostPrepare_PgStat();
@@ -2951,6 +2958,7 @@ AbortTransaction(void)
false, true);
AtEOXact_Buffers(false);
AtEOXact_RelationCache(false);
+ AtEOXact_TypeCache();
AtEOXact_Inval(false);
AtEOXact_MultiXact();
ResourceOwnerRelease(TopTransactionResourceOwner,
@@ -5153,6 +5161,7 @@ CommitSubTransaction(void)
true, false);
AtEOSubXact_RelationCache(true, s->subTransactionId,
s->parent->subTransactionId);
+ AtEOSubXact_TypeCache();
AtEOSubXact_Inval(true);
AtSubCommit_smgr();
@@ -5328,6 +5337,7 @@ AbortSubTransaction(void)
AtEOSubXact_RelationCache(false, s->subTransactionId,
s->parent->subTransactionId);
+ AtEOSubXact_TypeCache();
AtEOSubXact_Inval(false);
ResourceOwnerRelease(s->curTransactionOwner,
RESOURCE_RELEASE_LOCKS,
diff --git a/src/backend/utils/cache/typcache.c b/src/backend/utils/cache/typcache.c
index 11382547ec0..094d3ca00c1 100644
--- a/src/backend/utils/cache/typcache.c
+++ b/src/backend/utils/cache/typcache.c
@@ -66,6 +66,7 @@
#include "utils/builtins.h"
#include "utils/catcache.h"
#include "utils/fmgroids.h"
+#include "utils/injection_point.h"
#include "utils/inval.h"
#include "utils/lsyscache.h"
#include "utils/memutils.h"
@@ -77,6 +78,20 @@
/* The main type cache hashtable searched by lookup_type_cache */
static HTAB *TypeCacheHash = NULL;
+/*
+ * The mapping of relation's OID to the corresponding composite type OID.
+ * We're keeping the map entry when the corresponding typentry has something
+ * to clear i.e it has either TCFLAGS_HAVE_PG_TYPE_DATA, or
+ * TCFLAGS_OPERATOR_FLAGS, or tupdesc.
+ */
+static HTAB *RelIdToTypeIdCacheHash = NULL;
+
+typedef struct RelIdToTypeIdCacheEntry
+{
+ Oid relid; /* OID of the relation */
+ Oid composite_typid; /* OID of the relation's composite type */
+} RelIdToTypeIdCacheEntry;
+
/* List of type cache entries for domain types */
static TypeCacheEntry *firstDomainTypeEntry = NULL;
@@ -208,6 +223,10 @@ typedef struct SharedTypmodTableEntry
dsa_pointer shared_tupdesc;
} SharedTypmodTableEntry;
+static Oid *in_progress_list;
+static int in_progress_list_len;
+static int in_progress_list_maxlen;
+
/*
* A comparator function for SharedRecordTableKey.
*/
@@ -329,6 +348,8 @@ static void shared_record_typmod_registry_detach(dsm_segment *segment,
static TupleDesc find_or_make_matching_shared_tupledesc(TupleDesc tupdesc);
static dsa_pointer share_tupledesc(dsa_area *area, TupleDesc tupdesc,
uint32 typmod);
+static void insert_rel_type_cache_if_needed(TypeCacheEntry *typentry);
+static void delete_rel_type_cache_if_needed(TypeCacheEntry *typentry);
/*
@@ -366,11 +387,13 @@ lookup_type_cache(Oid type_id, int flags)
{
TypeCacheEntry *typentry;
bool found;
+ int in_progress_offset;
if (TypeCacheHash == NULL)
{
/* First time through: initialize the hash table */
HASHCTL ctl;
+ int allocsize;
ctl.keysize = sizeof(Oid);
ctl.entrysize = sizeof(TypeCacheEntry);
@@ -385,6 +408,13 @@ lookup_type_cache(Oid type_id, int flags)
TypeCacheHash = hash_create("Type information cache", 64,
&ctl, HASH_ELEM | HASH_FUNCTION);
+ Assert(RelIdToTypeIdCacheHash == NULL);
+
+ ctl.keysize = sizeof(Oid);
+ ctl.entrysize = sizeof(RelIdToTypeIdCacheEntry);
+ RelIdToTypeIdCacheHash = hash_create("Map from relid to OID of cached composite type", 64,
+ &ctl, HASH_ELEM | HASH_BLOBS);
+
/* Also set up callbacks for SI invalidations */
CacheRegisterRelcacheCallback(TypeCacheRelCallback, (Datum) 0);
CacheRegisterSyscacheCallback(TYPEOID, TypeCacheTypCallback, (Datum) 0);
@@ -394,8 +424,32 @@ lookup_type_cache(Oid type_id, int flags)
/* Also make sure CacheMemoryContext exists */
if (!CacheMemoryContext)
CreateCacheMemoryContext();
+
+ /*
+ * reserve enough in_progress_list slots for many cases
+ */
+ allocsize = 4;
+ in_progress_list =
+ MemoryContextAlloc(CacheMemoryContext,
+ allocsize * sizeof(*in_progress_list));
+ in_progress_list_maxlen = allocsize;
}
+ Assert(TypeCacheHash != NULL && RelIdToTypeIdCacheHash != NULL);
+
+ /* Register to catch invalidation messages */
+ if (in_progress_list_len >= in_progress_list_maxlen)
+ {
+ int allocsize;
+
+ allocsize = in_progress_list_maxlen * 2;
+ in_progress_list = repalloc(in_progress_list,
+ allocsize * sizeof(*in_progress_list));
+ in_progress_list_maxlen = allocsize;
+ }
+ in_progress_offset = in_progress_list_len++;
+ in_progress_list[in_progress_offset] = type_id;
+
/* Try to look up an existing entry */
typentry = (TypeCacheEntry *) hash_search(TypeCacheHash,
&type_id,
@@ -896,6 +950,13 @@ lookup_type_cache(Oid type_id, int flags)
load_domaintype_info(typentry);
}
+ INJECTION_POINT("typecache-before-rel-type-cache-insert");
+
+ Assert(in_progress_offset + 1 == in_progress_list_len);
+ in_progress_list_len--;
+
+ insert_rel_type_cache_if_needed(typentry);
+
return typentry;
}
@@ -2290,6 +2351,53 @@ SharedRecordTypmodRegistryAttach(SharedRecordTypmodRegistry *registry)
CurrentSession->shared_typmod_table = typmod_table;
}
+/*
+ * InvalidateCompositeTypeCacheEntry
+ * Invalidate particular TypeCacheEntry on Relcache inval callback
+ *
+ * Delete the cached tuple descriptor (if any) for the given composite
+ * type, and reset whatever info we have cached about the composite type's
+ * comparability.
+ */
+static void
+InvalidateCompositeTypeCacheEntry(TypeCacheEntry *typentry)
+{
+ bool hadTupDescOrOpclass;
+
+ Assert(typentry->typtype == TYPTYPE_COMPOSITE &&
+ OidIsValid(typentry->typrelid));
+
+ hadTupDescOrOpclass = (typentry->tupDesc != NULL) ||
+ (typentry->flags & TCFLAGS_OPERATOR_FLAGS);
+
+ /* Delete tupdesc if we have it */
+ if (typentry->tupDesc != NULL)
+ {
+ /*
+ * Release our refcount and free the tupdesc if none remain. We can't
+ * use DecrTupleDescRefCount here because this reference is not logged
+ * by the current resource owner.
+ */
+ Assert(typentry->tupDesc->tdrefcount > 0);
+ if (--typentry->tupDesc->tdrefcount == 0)
+ FreeTupleDesc(typentry->tupDesc);
+ typentry->tupDesc = NULL;
+
+ /*
+ * Also clear tupDesc_identifier, so that anyone watching it will
+ * realize that the tupdesc has changed.
+ */
+ typentry->tupDesc_identifier = 0;
+ }
+
+ /* Reset equality/comparison/hashing validity information */
+ typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
+
+ /* Call delete_rel_type_cache() if we actually cleared something */
+ if (hadTupDescOrOpclass)
+ delete_rel_type_cache_if_needed(typentry);
+}
+
/*
* TypeCacheRelCallback
* Relcache inval callback function
@@ -2299,63 +2407,55 @@ SharedRecordTypmodRegistryAttach(SharedRecordTypmodRegistry *registry)
* whatever info we have cached about the composite type's comparability.
*
* This is called when a relcache invalidation event occurs for the given
- * relid. We must scan the whole typcache hash since we don't know the
- * type OID corresponding to the relid. We could do a direct search if this
- * were a syscache-flush callback on pg_type, but then we would need all
- * ALTER-TABLE-like commands that could modify a rowtype to issue syscache
- * invals against the rel's pg_type OID. The extra SI signaling could very
- * well cost more than we'd save, since in most usages there are not very
- * many entries in a backend's typcache. The risk of bugs-of-omission seems
- * high, too.
- *
- * Another possibility, with only localized impact, is to maintain a second
- * hashtable that indexes composite-type typcache entries by their typrelid.
- * But it's still not clear it's worth the trouble.
+ * relid. We can't use syscache to find a type corresponding to the given
+ * relation because the code can be called outside of transaction. Thus we use
+ * the RelIdToTypeIdCacheHash map to locate appropriate typcache entry.
*/
static void
TypeCacheRelCallback(Datum arg, Oid relid)
{
- HASH_SEQ_STATUS status;
TypeCacheEntry *typentry;
- /* TypeCacheHash must exist, else this callback wouldn't be registered */
- hash_seq_init(&status, TypeCacheHash);
- while ((typentry = (TypeCacheEntry *) hash_seq_search(&status)) != NULL)
+ /*
+ * RelIdToTypeIdCacheHash and TypeCacheHash should exist, otherwise this
+ * callback wouldn't be registered
+ */
+ if (OidIsValid(relid))
{
- if (typentry->typtype == TYPTYPE_COMPOSITE)
+ RelIdToTypeIdCacheEntry *relentry;
+
+ /*
+ * Find an RelIdToTypeIdCacheHash entry, which should exist as soon as
+ * corresponding typcache entry has something to clean.
+ */
+ relentry = (RelIdToTypeIdCacheEntry *) hash_search(RelIdToTypeIdCacheHash,
+ &relid,
+ HASH_FIND, NULL);
+
+ if (relentry != NULL)
{
- /* Skip if no match, unless we're zapping all composite types */
- if (relid != typentry->typrelid && relid != InvalidOid)
- continue;
+ typentry = (TypeCacheEntry *) hash_search(TypeCacheHash,
+ &relentry->composite_typid,
+ HASH_FIND, NULL);
- /* Delete tupdesc if we have it */
- if (typentry->tupDesc != NULL)
+ if (typentry != NULL)
{
- /*
- * Release our refcount, and free the tupdesc if none remain.
- * (Can't use DecrTupleDescRefCount because this reference is
- * not logged in current resource owner.)
- */
- Assert(typentry->tupDesc->tdrefcount > 0);
- if (--typentry->tupDesc->tdrefcount == 0)
- FreeTupleDesc(typentry->tupDesc);
- typentry->tupDesc = NULL;
+ Assert(typentry->typtype == TYPTYPE_COMPOSITE);
+ Assert(relid == typentry->typrelid);
- /*
- * Also clear tupDesc_identifier, so that anything watching
- * that will realize that the tupdesc has possibly changed.
- * (Alternatively, we could specify that to detect possible
- * tupdesc change, one must check for tupDesc != NULL as well
- * as tupDesc_identifier being the same as what was previously
- * seen. That seems error-prone.)
- */
- typentry->tupDesc_identifier = 0;
+ InvalidateCompositeTypeCacheEntry(typentry);
}
-
- /* Reset equality/comparison/hashing validity information */
- typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
}
- else if (typentry->typtype == TYPTYPE_DOMAIN)
+
+ /*
+ * Visit all the domain types sequentially. Typically this shouldn't
+ * affect performance since domain types are less tended to bloat.
+ * Domain types are created manually, unlike composite types which are
+ * automatically created for every temporary table.
+ */
+ for (typentry = firstDomainTypeEntry;
+ typentry != NULL;
+ typentry = typentry->nextDomain)
{
/*
* If it's domain over composite, reset flags. (We don't bother
@@ -2367,6 +2467,36 @@ TypeCacheRelCallback(Datum arg, Oid relid)
typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
}
}
+ else
+ {
+ HASH_SEQ_STATUS status;
+
+ /*
+ * Relid is invalid. By convention we need to reset all composite
+ * types in cache. Also, we should reset flags for domain types, and
+ * we loop over all entries in hash, so, do it in a single scan.
+ */
+ hash_seq_init(&status, TypeCacheHash);
+ while ((typentry = (TypeCacheEntry *) hash_seq_search(&status)) != NULL)
+ {
+ if (typentry->typtype == TYPTYPE_COMPOSITE)
+ {
+ InvalidateCompositeTypeCacheEntry(typentry);
+ }
+ else if (typentry->typtype == TYPTYPE_DOMAIN)
+ {
+ /*
+ * If it's domain over composite, reset flags. (We don't
+ * bother trying to determine whether the specific base type
+ * needs a reset.) Note that if we haven't determined whether
+ * the base type is composite, we don't need to reset
+ * anything.
+ */
+ if (typentry->flags & TCFLAGS_DOMAIN_BASE_IS_COMPOSITE)
+ typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
+ }
+ }
+ }
}
/*
@@ -2397,6 +2527,8 @@ TypeCacheTypCallback(Datum arg, int cacheid, uint32 hashvalue)
while ((typentry = (TypeCacheEntry *) hash_seq_search(&status)) != NULL)
{
+ bool hadPgTypeData = (typentry->flags & TCFLAGS_HAVE_PG_TYPE_DATA);
+
Assert(hashvalue == 0 || typentry->type_id_hash == hashvalue);
/*
@@ -2406,6 +2538,13 @@ TypeCacheTypCallback(Datum arg, int cacheid, uint32 hashvalue)
*/
typentry->flags &= ~(TCFLAGS_HAVE_PG_TYPE_DATA |
TCFLAGS_CHECKED_DOMAIN_CONSTRAINTS);
+
+ /*
+ * Call delete_rel_type_cache() if we cleaned
+ * TCFLAGS_HAVE_PG_TYPE_DATA flag previously.
+ */
+ if (hadPgTypeData)
+ delete_rel_type_cache_if_needed(typentry);
}
}
@@ -2914,3 +3053,135 @@ shared_record_typmod_registry_detach(dsm_segment *segment, Datum datum)
}
CurrentSession->shared_typmod_registry = NULL;
}
+
+/*
+ * Insert RelIdToTypeIdCacheHash entry if needed.
+ */
+static void
+insert_rel_type_cache_if_needed(TypeCacheEntry *typentry)
+{
+ /* Immediately quit for non-composite types */
+ if (typentry->typtype != TYPTYPE_COMPOSITE)
+ return;
+
+ /* typrelid should be given for composite types */
+ Assert(OidIsValid(typentry->typrelid));
+
+ /*
+ * Insert a RelIdToTypeIdCacheHash entry if the typentry have any
+ * information indicating it should be here.
+ */
+ if ((typentry->flags & TCFLAGS_HAVE_PG_TYPE_DATA) ||
+ (typentry->flags & TCFLAGS_OPERATOR_FLAGS) ||
+ typentry->tupDesc != NULL)
+ {
+ RelIdToTypeIdCacheEntry *relentry;
+ bool found;
+
+ relentry = (RelIdToTypeIdCacheEntry *) hash_search(RelIdToTypeIdCacheHash,
+ &typentry->typrelid,
+ HASH_ENTER, &found);
+ relentry->relid = typentry->typrelid;
+ relentry->composite_typid = typentry->type_id;
+ }
+}
+
+/*
+ * Delete entry RelIdToTypeIdCacheHash if needed after resetting of the
+ * TCFLAGS_HAVE_PG_TYPE_DATA flag, or any of TCFLAGS_OPERATOR_FLAGS flags,
+ * or tupDesc.
+ */
+static void
+delete_rel_type_cache_if_needed(TypeCacheEntry *typentry)
+{
+#ifdef USE_ASSERT_CHECKING
+ int i;
+ bool is_in_progress = false;
+
+ for (i = 0; i < in_progress_list_len; i++)
+ {
+ if (in_progress_list[i] == typentry->type_id)
+ {
+ is_in_progress = true;
+ break;
+ }
+ }
+#endif
+
+ /* Immediately quit for non-composite types */
+ if (typentry->typtype != TYPTYPE_COMPOSITE)
+ return;
+
+ /* typrelid should be given for composite types */
+ Assert(OidIsValid(typentry->typrelid));
+
+ /*
+ * Delete a RelIdToTypeIdCacheHash entry if the typentry doesn't have any
+ * information indicating entry should be still there.
+ */
+ if (!(typentry->flags & TCFLAGS_HAVE_PG_TYPE_DATA) &&
+ !(typentry->flags & TCFLAGS_OPERATOR_FLAGS) &&
+ typentry->tupDesc == NULL)
+ {
+ bool found;
+
+ (void) hash_search(RelIdToTypeIdCacheHash,
+ &typentry->typrelid,
+ HASH_REMOVE, &found);
+ Assert(found || is_in_progress);
+ }
+ else
+ {
+#ifdef USE_ASSERT_CHECKING
+ /*
+ * In assert-enabled builds otherwise check for RelIdToTypeIdCacheHash
+ * entry if it should exist.
+ */
+ bool found;
+
+ if (!is_in_progress)
+ {
+ (void) hash_search(RelIdToTypeIdCacheHash,
+ &typentry->typrelid,
+ HASH_FIND, &found);
+ Assert(found);
+ }
+#endif
+ }
+}
+
+/*
+ * Add possibly missing RelIdToTypeId entries related to TypeCacheHash
+ * entries, marked as in-progress by lookup_type_cache(). It may happen
+ * in case of an error or interruption during the lookup_type_cache() call.
+ */
+static void
+finalize_in_progress_typentries(void)
+{
+ int i;
+
+ for (i = 0; i < in_progress_list_len; i++)
+ {
+ TypeCacheEntry *typentry;
+
+ typentry = (TypeCacheEntry *) hash_search(TypeCacheHash,
+ &in_progress_list[i],
+ HASH_FIND, NULL);
+ if (typentry)
+ insert_rel_type_cache_if_needed(typentry);
+ }
+
+ in_progress_list_len = 0;
+}
+
+void
+AtEOXact_TypeCache(void)
+{
+ finalize_in_progress_typentries();
+}
+
+void
+AtEOSubXact_TypeCache(void)
+{
+ finalize_in_progress_typentries();
+}
diff --git a/src/include/utils/typcache.h b/src/include/utils/typcache.h
index f506cc4aa35..f3d73ecee3a 100644
--- a/src/include/utils/typcache.h
+++ b/src/include/utils/typcache.h
@@ -207,4 +207,8 @@ extern void SharedRecordTypmodRegistryInit(SharedRecordTypmodRegistry *,
extern void SharedRecordTypmodRegistryAttach(SharedRecordTypmodRegistry *);
+extern void AtEOXact_TypeCache(void);
+
+extern void AtEOSubXact_TypeCache(void);
+
#endif /* TYPCACHE_H */
diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile
index 256799f520a..c0d3cf0e14b 100644
--- a/src/test/modules/Makefile
+++ b/src/test/modules/Makefile
@@ -43,9 +43,9 @@ SUBDIRS = \
ifeq ($(enable_injection_points),yes)
-SUBDIRS += injection_points gin
+SUBDIRS += injection_points gin typcache
else
-ALWAYS_SUBDIRS += injection_points gin
+ALWAYS_SUBDIRS += injection_points gin typcache
endif
ifeq ($(with_ssl),openssl)
diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build
index d8fe059d236..c829b619530 100644
--- a/src/test/modules/meson.build
+++ b/src/test/modules/meson.build
@@ -36,6 +36,7 @@ subdir('test_rls_hooks')
subdir('test_shm_mq')
subdir('test_slru')
subdir('test_tidstore')
+subdir('typcache')
subdir('unsafe_tests')
subdir('worker_spi')
subdir('xid_wraparound')
diff --git a/src/test/modules/typcache/.gitignore b/src/test/modules/typcache/.gitignore
new file mode 100644
index 00000000000..5dcb3ff9723
--- /dev/null
+++ b/src/test/modules/typcache/.gitignore
@@ -0,0 +1,4 @@
+# Generated subdirectories
+/log/
+/results/
+/tmp_check/
diff --git a/src/test/modules/typcache/Makefile b/src/test/modules/typcache/Makefile
new file mode 100644
index 00000000000..6ee46ec0891
--- /dev/null
+++ b/src/test/modules/typcache/Makefile
@@ -0,0 +1,28 @@
+# src/test/modules/typcache/Makefile
+
+EXTRA_INSTALL = src/test/modules/typcache_rel_type_cache.out.out
+
+REGRESS = typcache_rel_type_cache
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = src/test/modules/typcache
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+
+# XXX: This test is conditional on enable_injection_points in the
+# parent Makefile, so we should never get here in the first place if
+# injection points are not enabled. But the buildfarm 'misc-check'
+# step doesn't pay attention to the if-condition in the parent
+# Makefile. To work around that, disable running the test here too.
+ifeq ($(enable_injection_points),yes)
+include $(top_srcdir)/contrib/contrib-global.mk
+else
+check:
+ @echo "injection points are disabled in this build"
+endif
+
+endif
diff --git a/src/test/modules/typcache/expected/typcache_rel_type_cache.out b/src/test/modules/typcache/expected/typcache_rel_type_cache.out
new file mode 100644
index 00000000000..b113e0bbd5d
--- /dev/null
+++ b/src/test/modules/typcache/expected/typcache_rel_type_cache.out
@@ -0,0 +1,34 @@
+--
+-- This test checks that lookup_type_cache() can correctly handle an
+-- interruption. We use the injection point to simulate an error but note
+-- that a similar situation could happen due to user query interruption.
+-- Despite the interruption, a map entry from the relation oid to type cache
+-- entry should be created. This is validated by subsequent modification of
+-- the table schema, then type casts which use new schema implying
+-- successful type cache invalidation by relation oid.
+--
+CREATE EXTENSION injection_points;
+CREATE TABLE t (i int);
+SELECT injection_points_attach('typecache-before-rel-type-cache-insert', 'error');
+ injection_points_attach
+-------------------------
+
+(1 row)
+
+SELECT '(1)'::t;
+ERROR: error triggered for injection point typecache-before-rel-type-cache-insert
+LINE 1: SELECT '(1)'::t;
+ ^
+SELECT injection_points_detach('typecache-before-rel-type-cache-insert');
+ injection_points_detach
+-------------------------
+
+(1 row)
+
+ALTER TABLE t ADD COLUMN j int;
+SELECT '(1,2)'::t;
+ t
+-------
+ (1,2)
+(1 row)
+
diff --git a/src/test/modules/typcache/meson.build b/src/test/modules/typcache/meson.build
new file mode 100644
index 00000000000..cb2e34c0d2b
--- /dev/null
+++ b/src/test/modules/typcache/meson.build
@@ -0,0 +1,16 @@
+# Copyright (c) 2022-2024, PostgreSQL Global Development Group
+
+if not get_option('injection_points')
+ subdir_done()
+endif
+
+tests += {
+ 'name': 'typcache',
+ 'sd': meson.current_source_dir(),
+ 'bd': meson.current_build_dir(),
+ 'regress': {
+ 'sql': [
+ 'typcache_rel_type_cache',
+ ],
+ },
+}
diff --git a/src/test/modules/typcache/sql/typcache_rel_type_cache.sql b/src/test/modules/typcache/sql/typcache_rel_type_cache.sql
new file mode 100644
index 00000000000..2c0a434d988
--- /dev/null
+++ b/src/test/modules/typcache/sql/typcache_rel_type_cache.sql
@@ -0,0 +1,18 @@
+--
+-- This test checks that lookup_type_cache() can correctly handle an
+-- interruption. We use the injection point to simulate an error but note
+-- that a similar situation could happen due to user query interruption.
+-- Despite the interruption, a map entry from the relation oid to type cache
+-- entry should be created. This is validated by subsequent modification of
+-- the table schema, then type casts which use new schema implying
+-- successful type cache invalidation by relation oid.
+--
+
+CREATE EXTENSION injection_points;
+
+CREATE TABLE t (i int);
+SELECT injection_points_attach('typecache-before-rel-type-cache-insert', 'error');
+SELECT '(1)'::t;
+SELECT injection_points_detach('typecache-before-rel-type-cache-insert');
+ALTER TABLE t ADD COLUMN j int;
+SELECT '(1,2)'::t;
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 57de1acff3a..9b3e7fd104b 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2378,6 +2378,7 @@ RelFileLocator
RelFileLocatorBackend
RelFileNumber
RelIdCacheEnt
+RelIdToTypeIdCacheEntry
RelInfo
RelInfoArr
RelMapFile
--
2.39.5 (Apple Git-154)
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2024-10-21 10:16 Dagfinn Ilmari Mannsåker <[email protected]>
parent: Alexander Korotkov <[email protected]>
0 siblings, 1 reply; 55+ messages in thread
From: Dagfinn Ilmari Mannsåker @ 2024-10-21 10:16 UTC (permalink / raw)
To: Alexander Korotkov <[email protected]>; +Cc: Andrei Lepikhov <[email protected]>; jian he <[email protected]>; Artur Zakirov <[email protected]>; Alexander Lakhin <[email protected]>; Pavel Borisov <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
Alexander Korotkov <[email protected]> writes:
> On Mon, Oct 21, 2024 at 8:40 AM Andrei Lepikhov <[email protected]> wrote:
>>
>> On 21/10/2024 06:32, Dagfinn Ilmari Mannsåker wrote:
>> > Alexander Korotkov <[email protected]> writes:
>> >
>> >> +static Oid *in_progress_list;
>> >> +static int in_progress_list_len;
>> >> +static int in_progress_list_maxlen;
>> >
>> > Is there any particular reason not to use pg_list.h for this?
>> Sure. The type cache lookup has to be as much optimal as possible.
>> Using an array and relating sequential access to it, we avoid memory
>> allocations and deallocations 99.9% of the time. Also, quick access to
>> the single element (which we will have in real life almost all of the
>> time) is much faster than employing list machinery.
Lists are actually dynamically resized arrays these days (see commit
1cff1b95ab6ddae32faa3efe0d95a820dbfdc164), not linked lists, so
accessing arbitrary elements is O(1), not O(n). Just like this patch,
the size is doubled (starting at 16) whenever array is full.
> +1,
> List with zero elements has to be NIL. That means continuous
> allocations/deallocations.
This however is a valid point (unless we keep a dummy zeroth element to
avoid it, which is even uglier than open-coding the array extension
logic), so objection withdrawn.
- ilmari
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2024-10-21 11:30 Alexander Korotkov <[email protected]>
parent: Dagfinn Ilmari Mannsåker <[email protected]>
0 siblings, 1 reply; 55+ messages in thread
From: Alexander Korotkov @ 2024-10-21 11:30 UTC (permalink / raw)
To: Dagfinn Ilmari Mannsåker <[email protected]>; +Cc: Andrei Lepikhov <[email protected]>; jian he <[email protected]>; Artur Zakirov <[email protected]>; Alexander Lakhin <[email protected]>; Pavel Borisov <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
On Mon, Oct 21, 2024 at 1:16 PM Dagfinn Ilmari Mannsåker
<[email protected]> wrote:
> Alexander Korotkov <[email protected]> writes:
>
> > On Mon, Oct 21, 2024 at 8:40 AM Andrei Lepikhov <[email protected]> wrote:
> >>
> >> On 21/10/2024 06:32, Dagfinn Ilmari Mannsåker wrote:
> >> > Alexander Korotkov <[email protected]> writes:
> >> >
> >> >> +static Oid *in_progress_list;
> >> >> +static int in_progress_list_len;
> >> >> +static int in_progress_list_maxlen;
> >> >
> >> > Is there any particular reason not to use pg_list.h for this?
> >> Sure. The type cache lookup has to be as much optimal as possible.
> >> Using an array and relating sequential access to it, we avoid memory
> >> allocations and deallocations 99.9% of the time. Also, quick access to
> >> the single element (which we will have in real life almost all of the
> >> time) is much faster than employing list machinery.
>
> Lists are actually dynamically resized arrays these days (see commit
> 1cff1b95ab6ddae32faa3efe0d95a820dbfdc164), not linked lists, so
> accessing arbitrary elements is O(1), not O(n). Just like this patch,
> the size is doubled (starting at 16) whenever array is full.
>
> > +1,
> > List with zero elements has to be NIL. That means continuous
> > allocations/deallocations.
>
> This however is a valid point (unless we keep a dummy zeroth element to
> avoid it, which is even uglier than open-coding the array extension
> logic), so objection withdrawn.
OK, thank you!
The attached revision fixes EXTRA_INSTALL in
src/test/modules/typcache/Makefile. Spotted off-list by Arthur
Zakirov.
------
Regards,
Alexander Korotkov
Supabase
Attachments:
[application/octet-stream] v17-0001-Update-header-comment-for-lookup_type_cache.patch (1.4K, ../../CAPpHfduqZFZYL63Yx92pfAyiS8ACXyXSsnV1Geb_ecJTTU=Pmg@mail.gmail.com/2-v17-0001-Update-header-comment-for-lookup_type_cache.patch)
download | inline diff:
From d624b14c9a4bd93426a5e475d4503589927ef2ae Mon Sep 17 00:00:00 2001
From: Alexander Korotkov <[email protected]>
Date: Fri, 13 Sep 2024 02:10:04 +0300
Subject: [PATCH v17 1/2] Update header comment for lookup_type_cache()
Describe the way we handle concurrent invalidation messages.
---
src/backend/utils/cache/typcache.c | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/src/backend/utils/cache/typcache.c b/src/backend/utils/cache/typcache.c
index 2ec136b7d30..11382547ec0 100644
--- a/src/backend/utils/cache/typcache.c
+++ b/src/backend/utils/cache/typcache.c
@@ -351,6 +351,15 @@ type_cache_syshash(const void *key, Size keysize)
* invalid. Note however that we may fail to find one or more of the
* values requested by 'flags'; the caller needs to check whether the fields
* are InvalidOid or not.
+ *
+ * Note that while filling TypeCacheEntry we might process concurrent
+ * invalidation messages, causing our not-yet-filled TypeCacheEntry to be
+ * invalidated. In this case, we typically only clear flags while values are
+ * still available for the caller. It's expected that the caller holds
+ * enough locks on type-depending objects that the values are still relevant.
+ * It's also important that the tupdesc is filled in the last for
+ * TYPTYPE_COMPOSITE. So, it can't get invalidated during the
+ * lookup_type_cache() call.
*/
TypeCacheEntry *
lookup_type_cache(Oid type_id, int flags)
--
2.39.5 (Apple Git-154)
[application/octet-stream] v17-0002-Avoid-looping-over-all-type-cache-entries-in-Typ.patch (24.7K, ../../CAPpHfduqZFZYL63Yx92pfAyiS8ACXyXSsnV1Geb_ecJTTU=Pmg@mail.gmail.com/3-v17-0002-Avoid-looping-over-all-type-cache-entries-in-Typ.patch)
download | inline diff:
From 41da564395bd147939d425657caacfab572a1fce Mon Sep 17 00:00:00 2001
From: Alexander Korotkov <[email protected]>
Date: Tue, 10 Sep 2024 23:25:04 +0300
Subject: [PATCH v17 2/2] Avoid looping over all type cache entries in
TypeCacheRelCallback()
Currently when a single relcache entry gets invalidated,
TypeCacheRelCallback() has to loop over all type cache entries to find
appropriate typentry to invalidate. Unfortunately, using the syscache here
is impossible, because this callback could be called outside a transaction
and this makes impossible catalog lookups. This is why present commit
introduces RelIdToTypeIdCacheHash to map relation OID to its composite type
OID.
We are keeping RelIdToTypeIdCacheHash entry while corresponding type cache
entry have something to clean. Therefore, RelIdToTypeIdCacheHash shouldn't
get bloat in the case of temporary tables flood.
There are many places in lookup_type_cache() where syscache invalidation,
user interruption, or even error could occur. In order to handle this, we
keep an array of in-progress type cache entries. In the case of
lookup_type_cache() interruption this array is processed to keep
RelIdToTypeIdCacheHash in a consistent state.
Discussion: https://postgr.es/m/5812a6e5-68ae-4d84-9d85-b443176966a1%40sigaev.ru
Author: Teodor Sigaev
Reviewed-by: Aleksander Alekseev, Tom Lane, Michael Paquier, Roman Zharkov
Reviewed-by: Andrei Lepikhov, Pavel Borisov, Jian He, Alexander Lakhin
Reviewed-by: Artur Zakirov
---
src/backend/access/transam/xact.c | 10 +
src/backend/utils/cache/typcache.c | 359 +++++++++++++++---
src/include/utils/typcache.h | 4 +
src/test/modules/Makefile | 4 +-
src/test/modules/meson.build | 1 +
src/test/modules/typcache/.gitignore | 4 +
src/test/modules/typcache/Makefile | 28 ++
.../expected/typcache_rel_type_cache.out | 34 ++
src/test/modules/typcache/meson.build | 16 +
.../typcache/sql/typcache_rel_type_cache.sql | 18 +
src/tools/pgindent/typedefs.list | 1 +
11 files changed, 433 insertions(+), 46 deletions(-)
create mode 100644 src/test/modules/typcache/.gitignore
create mode 100644 src/test/modules/typcache/Makefile
create mode 100644 src/test/modules/typcache/expected/typcache_rel_type_cache.out
create mode 100644 src/test/modules/typcache/meson.build
create mode 100644 src/test/modules/typcache/sql/typcache_rel_type_cache.sql
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 87700c7c5c7..b0b05e28790 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -70,6 +70,7 @@
#include "utils/snapmgr.h"
#include "utils/timeout.h"
#include "utils/timestamp.h"
+#include "utils/typcache.h"
/*
* User-tweakable parameters
@@ -2407,6 +2408,9 @@ CommitTransaction(void)
/* Clean up the relation cache */
AtEOXact_RelationCache(true);
+ /* Clean up the type cache */
+ AtEOXact_TypeCache();
+
/*
* Make catalog changes visible to all backends. This has to happen after
* relcache references are dropped (see comments for
@@ -2709,6 +2713,9 @@ PrepareTransaction(void)
/* Clean up the relation cache */
AtEOXact_RelationCache(true);
+ /* Clean up the type cache */
+ AtEOXact_TypeCache();
+
/* notify doesn't need a postprepare call */
PostPrepare_PgStat();
@@ -2951,6 +2958,7 @@ AbortTransaction(void)
false, true);
AtEOXact_Buffers(false);
AtEOXact_RelationCache(false);
+ AtEOXact_TypeCache();
AtEOXact_Inval(false);
AtEOXact_MultiXact();
ResourceOwnerRelease(TopTransactionResourceOwner,
@@ -5153,6 +5161,7 @@ CommitSubTransaction(void)
true, false);
AtEOSubXact_RelationCache(true, s->subTransactionId,
s->parent->subTransactionId);
+ AtEOSubXact_TypeCache();
AtEOSubXact_Inval(true);
AtSubCommit_smgr();
@@ -5328,6 +5337,7 @@ AbortSubTransaction(void)
AtEOSubXact_RelationCache(false, s->subTransactionId,
s->parent->subTransactionId);
+ AtEOSubXact_TypeCache();
AtEOSubXact_Inval(false);
ResourceOwnerRelease(s->curTransactionOwner,
RESOURCE_RELEASE_LOCKS,
diff --git a/src/backend/utils/cache/typcache.c b/src/backend/utils/cache/typcache.c
index 11382547ec0..094d3ca00c1 100644
--- a/src/backend/utils/cache/typcache.c
+++ b/src/backend/utils/cache/typcache.c
@@ -66,6 +66,7 @@
#include "utils/builtins.h"
#include "utils/catcache.h"
#include "utils/fmgroids.h"
+#include "utils/injection_point.h"
#include "utils/inval.h"
#include "utils/lsyscache.h"
#include "utils/memutils.h"
@@ -77,6 +78,20 @@
/* The main type cache hashtable searched by lookup_type_cache */
static HTAB *TypeCacheHash = NULL;
+/*
+ * The mapping of relation's OID to the corresponding composite type OID.
+ * We're keeping the map entry when the corresponding typentry has something
+ * to clear i.e it has either TCFLAGS_HAVE_PG_TYPE_DATA, or
+ * TCFLAGS_OPERATOR_FLAGS, or tupdesc.
+ */
+static HTAB *RelIdToTypeIdCacheHash = NULL;
+
+typedef struct RelIdToTypeIdCacheEntry
+{
+ Oid relid; /* OID of the relation */
+ Oid composite_typid; /* OID of the relation's composite type */
+} RelIdToTypeIdCacheEntry;
+
/* List of type cache entries for domain types */
static TypeCacheEntry *firstDomainTypeEntry = NULL;
@@ -208,6 +223,10 @@ typedef struct SharedTypmodTableEntry
dsa_pointer shared_tupdesc;
} SharedTypmodTableEntry;
+static Oid *in_progress_list;
+static int in_progress_list_len;
+static int in_progress_list_maxlen;
+
/*
* A comparator function for SharedRecordTableKey.
*/
@@ -329,6 +348,8 @@ static void shared_record_typmod_registry_detach(dsm_segment *segment,
static TupleDesc find_or_make_matching_shared_tupledesc(TupleDesc tupdesc);
static dsa_pointer share_tupledesc(dsa_area *area, TupleDesc tupdesc,
uint32 typmod);
+static void insert_rel_type_cache_if_needed(TypeCacheEntry *typentry);
+static void delete_rel_type_cache_if_needed(TypeCacheEntry *typentry);
/*
@@ -366,11 +387,13 @@ lookup_type_cache(Oid type_id, int flags)
{
TypeCacheEntry *typentry;
bool found;
+ int in_progress_offset;
if (TypeCacheHash == NULL)
{
/* First time through: initialize the hash table */
HASHCTL ctl;
+ int allocsize;
ctl.keysize = sizeof(Oid);
ctl.entrysize = sizeof(TypeCacheEntry);
@@ -385,6 +408,13 @@ lookup_type_cache(Oid type_id, int flags)
TypeCacheHash = hash_create("Type information cache", 64,
&ctl, HASH_ELEM | HASH_FUNCTION);
+ Assert(RelIdToTypeIdCacheHash == NULL);
+
+ ctl.keysize = sizeof(Oid);
+ ctl.entrysize = sizeof(RelIdToTypeIdCacheEntry);
+ RelIdToTypeIdCacheHash = hash_create("Map from relid to OID of cached composite type", 64,
+ &ctl, HASH_ELEM | HASH_BLOBS);
+
/* Also set up callbacks for SI invalidations */
CacheRegisterRelcacheCallback(TypeCacheRelCallback, (Datum) 0);
CacheRegisterSyscacheCallback(TYPEOID, TypeCacheTypCallback, (Datum) 0);
@@ -394,8 +424,32 @@ lookup_type_cache(Oid type_id, int flags)
/* Also make sure CacheMemoryContext exists */
if (!CacheMemoryContext)
CreateCacheMemoryContext();
+
+ /*
+ * reserve enough in_progress_list slots for many cases
+ */
+ allocsize = 4;
+ in_progress_list =
+ MemoryContextAlloc(CacheMemoryContext,
+ allocsize * sizeof(*in_progress_list));
+ in_progress_list_maxlen = allocsize;
}
+ Assert(TypeCacheHash != NULL && RelIdToTypeIdCacheHash != NULL);
+
+ /* Register to catch invalidation messages */
+ if (in_progress_list_len >= in_progress_list_maxlen)
+ {
+ int allocsize;
+
+ allocsize = in_progress_list_maxlen * 2;
+ in_progress_list = repalloc(in_progress_list,
+ allocsize * sizeof(*in_progress_list));
+ in_progress_list_maxlen = allocsize;
+ }
+ in_progress_offset = in_progress_list_len++;
+ in_progress_list[in_progress_offset] = type_id;
+
/* Try to look up an existing entry */
typentry = (TypeCacheEntry *) hash_search(TypeCacheHash,
&type_id,
@@ -896,6 +950,13 @@ lookup_type_cache(Oid type_id, int flags)
load_domaintype_info(typentry);
}
+ INJECTION_POINT("typecache-before-rel-type-cache-insert");
+
+ Assert(in_progress_offset + 1 == in_progress_list_len);
+ in_progress_list_len--;
+
+ insert_rel_type_cache_if_needed(typentry);
+
return typentry;
}
@@ -2290,6 +2351,53 @@ SharedRecordTypmodRegistryAttach(SharedRecordTypmodRegistry *registry)
CurrentSession->shared_typmod_table = typmod_table;
}
+/*
+ * InvalidateCompositeTypeCacheEntry
+ * Invalidate particular TypeCacheEntry on Relcache inval callback
+ *
+ * Delete the cached tuple descriptor (if any) for the given composite
+ * type, and reset whatever info we have cached about the composite type's
+ * comparability.
+ */
+static void
+InvalidateCompositeTypeCacheEntry(TypeCacheEntry *typentry)
+{
+ bool hadTupDescOrOpclass;
+
+ Assert(typentry->typtype == TYPTYPE_COMPOSITE &&
+ OidIsValid(typentry->typrelid));
+
+ hadTupDescOrOpclass = (typentry->tupDesc != NULL) ||
+ (typentry->flags & TCFLAGS_OPERATOR_FLAGS);
+
+ /* Delete tupdesc if we have it */
+ if (typentry->tupDesc != NULL)
+ {
+ /*
+ * Release our refcount and free the tupdesc if none remain. We can't
+ * use DecrTupleDescRefCount here because this reference is not logged
+ * by the current resource owner.
+ */
+ Assert(typentry->tupDesc->tdrefcount > 0);
+ if (--typentry->tupDesc->tdrefcount == 0)
+ FreeTupleDesc(typentry->tupDesc);
+ typentry->tupDesc = NULL;
+
+ /*
+ * Also clear tupDesc_identifier, so that anyone watching it will
+ * realize that the tupdesc has changed.
+ */
+ typentry->tupDesc_identifier = 0;
+ }
+
+ /* Reset equality/comparison/hashing validity information */
+ typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
+
+ /* Call delete_rel_type_cache() if we actually cleared something */
+ if (hadTupDescOrOpclass)
+ delete_rel_type_cache_if_needed(typentry);
+}
+
/*
* TypeCacheRelCallback
* Relcache inval callback function
@@ -2299,63 +2407,55 @@ SharedRecordTypmodRegistryAttach(SharedRecordTypmodRegistry *registry)
* whatever info we have cached about the composite type's comparability.
*
* This is called when a relcache invalidation event occurs for the given
- * relid. We must scan the whole typcache hash since we don't know the
- * type OID corresponding to the relid. We could do a direct search if this
- * were a syscache-flush callback on pg_type, but then we would need all
- * ALTER-TABLE-like commands that could modify a rowtype to issue syscache
- * invals against the rel's pg_type OID. The extra SI signaling could very
- * well cost more than we'd save, since in most usages there are not very
- * many entries in a backend's typcache. The risk of bugs-of-omission seems
- * high, too.
- *
- * Another possibility, with only localized impact, is to maintain a second
- * hashtable that indexes composite-type typcache entries by their typrelid.
- * But it's still not clear it's worth the trouble.
+ * relid. We can't use syscache to find a type corresponding to the given
+ * relation because the code can be called outside of transaction. Thus we use
+ * the RelIdToTypeIdCacheHash map to locate appropriate typcache entry.
*/
static void
TypeCacheRelCallback(Datum arg, Oid relid)
{
- HASH_SEQ_STATUS status;
TypeCacheEntry *typentry;
- /* TypeCacheHash must exist, else this callback wouldn't be registered */
- hash_seq_init(&status, TypeCacheHash);
- while ((typentry = (TypeCacheEntry *) hash_seq_search(&status)) != NULL)
+ /*
+ * RelIdToTypeIdCacheHash and TypeCacheHash should exist, otherwise this
+ * callback wouldn't be registered
+ */
+ if (OidIsValid(relid))
{
- if (typentry->typtype == TYPTYPE_COMPOSITE)
+ RelIdToTypeIdCacheEntry *relentry;
+
+ /*
+ * Find an RelIdToTypeIdCacheHash entry, which should exist as soon as
+ * corresponding typcache entry has something to clean.
+ */
+ relentry = (RelIdToTypeIdCacheEntry *) hash_search(RelIdToTypeIdCacheHash,
+ &relid,
+ HASH_FIND, NULL);
+
+ if (relentry != NULL)
{
- /* Skip if no match, unless we're zapping all composite types */
- if (relid != typentry->typrelid && relid != InvalidOid)
- continue;
+ typentry = (TypeCacheEntry *) hash_search(TypeCacheHash,
+ &relentry->composite_typid,
+ HASH_FIND, NULL);
- /* Delete tupdesc if we have it */
- if (typentry->tupDesc != NULL)
+ if (typentry != NULL)
{
- /*
- * Release our refcount, and free the tupdesc if none remain.
- * (Can't use DecrTupleDescRefCount because this reference is
- * not logged in current resource owner.)
- */
- Assert(typentry->tupDesc->tdrefcount > 0);
- if (--typentry->tupDesc->tdrefcount == 0)
- FreeTupleDesc(typentry->tupDesc);
- typentry->tupDesc = NULL;
+ Assert(typentry->typtype == TYPTYPE_COMPOSITE);
+ Assert(relid == typentry->typrelid);
- /*
- * Also clear tupDesc_identifier, so that anything watching
- * that will realize that the tupdesc has possibly changed.
- * (Alternatively, we could specify that to detect possible
- * tupdesc change, one must check for tupDesc != NULL as well
- * as tupDesc_identifier being the same as what was previously
- * seen. That seems error-prone.)
- */
- typentry->tupDesc_identifier = 0;
+ InvalidateCompositeTypeCacheEntry(typentry);
}
-
- /* Reset equality/comparison/hashing validity information */
- typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
}
- else if (typentry->typtype == TYPTYPE_DOMAIN)
+
+ /*
+ * Visit all the domain types sequentially. Typically this shouldn't
+ * affect performance since domain types are less tended to bloat.
+ * Domain types are created manually, unlike composite types which are
+ * automatically created for every temporary table.
+ */
+ for (typentry = firstDomainTypeEntry;
+ typentry != NULL;
+ typentry = typentry->nextDomain)
{
/*
* If it's domain over composite, reset flags. (We don't bother
@@ -2367,6 +2467,36 @@ TypeCacheRelCallback(Datum arg, Oid relid)
typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
}
}
+ else
+ {
+ HASH_SEQ_STATUS status;
+
+ /*
+ * Relid is invalid. By convention we need to reset all composite
+ * types in cache. Also, we should reset flags for domain types, and
+ * we loop over all entries in hash, so, do it in a single scan.
+ */
+ hash_seq_init(&status, TypeCacheHash);
+ while ((typentry = (TypeCacheEntry *) hash_seq_search(&status)) != NULL)
+ {
+ if (typentry->typtype == TYPTYPE_COMPOSITE)
+ {
+ InvalidateCompositeTypeCacheEntry(typentry);
+ }
+ else if (typentry->typtype == TYPTYPE_DOMAIN)
+ {
+ /*
+ * If it's domain over composite, reset flags. (We don't
+ * bother trying to determine whether the specific base type
+ * needs a reset.) Note that if we haven't determined whether
+ * the base type is composite, we don't need to reset
+ * anything.
+ */
+ if (typentry->flags & TCFLAGS_DOMAIN_BASE_IS_COMPOSITE)
+ typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
+ }
+ }
+ }
}
/*
@@ -2397,6 +2527,8 @@ TypeCacheTypCallback(Datum arg, int cacheid, uint32 hashvalue)
while ((typentry = (TypeCacheEntry *) hash_seq_search(&status)) != NULL)
{
+ bool hadPgTypeData = (typentry->flags & TCFLAGS_HAVE_PG_TYPE_DATA);
+
Assert(hashvalue == 0 || typentry->type_id_hash == hashvalue);
/*
@@ -2406,6 +2538,13 @@ TypeCacheTypCallback(Datum arg, int cacheid, uint32 hashvalue)
*/
typentry->flags &= ~(TCFLAGS_HAVE_PG_TYPE_DATA |
TCFLAGS_CHECKED_DOMAIN_CONSTRAINTS);
+
+ /*
+ * Call delete_rel_type_cache() if we cleaned
+ * TCFLAGS_HAVE_PG_TYPE_DATA flag previously.
+ */
+ if (hadPgTypeData)
+ delete_rel_type_cache_if_needed(typentry);
}
}
@@ -2914,3 +3053,135 @@ shared_record_typmod_registry_detach(dsm_segment *segment, Datum datum)
}
CurrentSession->shared_typmod_registry = NULL;
}
+
+/*
+ * Insert RelIdToTypeIdCacheHash entry if needed.
+ */
+static void
+insert_rel_type_cache_if_needed(TypeCacheEntry *typentry)
+{
+ /* Immediately quit for non-composite types */
+ if (typentry->typtype != TYPTYPE_COMPOSITE)
+ return;
+
+ /* typrelid should be given for composite types */
+ Assert(OidIsValid(typentry->typrelid));
+
+ /*
+ * Insert a RelIdToTypeIdCacheHash entry if the typentry have any
+ * information indicating it should be here.
+ */
+ if ((typentry->flags & TCFLAGS_HAVE_PG_TYPE_DATA) ||
+ (typentry->flags & TCFLAGS_OPERATOR_FLAGS) ||
+ typentry->tupDesc != NULL)
+ {
+ RelIdToTypeIdCacheEntry *relentry;
+ bool found;
+
+ relentry = (RelIdToTypeIdCacheEntry *) hash_search(RelIdToTypeIdCacheHash,
+ &typentry->typrelid,
+ HASH_ENTER, &found);
+ relentry->relid = typentry->typrelid;
+ relentry->composite_typid = typentry->type_id;
+ }
+}
+
+/*
+ * Delete entry RelIdToTypeIdCacheHash if needed after resetting of the
+ * TCFLAGS_HAVE_PG_TYPE_DATA flag, or any of TCFLAGS_OPERATOR_FLAGS flags,
+ * or tupDesc.
+ */
+static void
+delete_rel_type_cache_if_needed(TypeCacheEntry *typentry)
+{
+#ifdef USE_ASSERT_CHECKING
+ int i;
+ bool is_in_progress = false;
+
+ for (i = 0; i < in_progress_list_len; i++)
+ {
+ if (in_progress_list[i] == typentry->type_id)
+ {
+ is_in_progress = true;
+ break;
+ }
+ }
+#endif
+
+ /* Immediately quit for non-composite types */
+ if (typentry->typtype != TYPTYPE_COMPOSITE)
+ return;
+
+ /* typrelid should be given for composite types */
+ Assert(OidIsValid(typentry->typrelid));
+
+ /*
+ * Delete a RelIdToTypeIdCacheHash entry if the typentry doesn't have any
+ * information indicating entry should be still there.
+ */
+ if (!(typentry->flags & TCFLAGS_HAVE_PG_TYPE_DATA) &&
+ !(typentry->flags & TCFLAGS_OPERATOR_FLAGS) &&
+ typentry->tupDesc == NULL)
+ {
+ bool found;
+
+ (void) hash_search(RelIdToTypeIdCacheHash,
+ &typentry->typrelid,
+ HASH_REMOVE, &found);
+ Assert(found || is_in_progress);
+ }
+ else
+ {
+#ifdef USE_ASSERT_CHECKING
+ /*
+ * In assert-enabled builds otherwise check for RelIdToTypeIdCacheHash
+ * entry if it should exist.
+ */
+ bool found;
+
+ if (!is_in_progress)
+ {
+ (void) hash_search(RelIdToTypeIdCacheHash,
+ &typentry->typrelid,
+ HASH_FIND, &found);
+ Assert(found);
+ }
+#endif
+ }
+}
+
+/*
+ * Add possibly missing RelIdToTypeId entries related to TypeCacheHash
+ * entries, marked as in-progress by lookup_type_cache(). It may happen
+ * in case of an error or interruption during the lookup_type_cache() call.
+ */
+static void
+finalize_in_progress_typentries(void)
+{
+ int i;
+
+ for (i = 0; i < in_progress_list_len; i++)
+ {
+ TypeCacheEntry *typentry;
+
+ typentry = (TypeCacheEntry *) hash_search(TypeCacheHash,
+ &in_progress_list[i],
+ HASH_FIND, NULL);
+ if (typentry)
+ insert_rel_type_cache_if_needed(typentry);
+ }
+
+ in_progress_list_len = 0;
+}
+
+void
+AtEOXact_TypeCache(void)
+{
+ finalize_in_progress_typentries();
+}
+
+void
+AtEOSubXact_TypeCache(void)
+{
+ finalize_in_progress_typentries();
+}
diff --git a/src/include/utils/typcache.h b/src/include/utils/typcache.h
index f506cc4aa35..f3d73ecee3a 100644
--- a/src/include/utils/typcache.h
+++ b/src/include/utils/typcache.h
@@ -207,4 +207,8 @@ extern void SharedRecordTypmodRegistryInit(SharedRecordTypmodRegistry *,
extern void SharedRecordTypmodRegistryAttach(SharedRecordTypmodRegistry *);
+extern void AtEOXact_TypeCache(void);
+
+extern void AtEOSubXact_TypeCache(void);
+
#endif /* TYPCACHE_H */
diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile
index 256799f520a..c0d3cf0e14b 100644
--- a/src/test/modules/Makefile
+++ b/src/test/modules/Makefile
@@ -43,9 +43,9 @@ SUBDIRS = \
ifeq ($(enable_injection_points),yes)
-SUBDIRS += injection_points gin
+SUBDIRS += injection_points gin typcache
else
-ALWAYS_SUBDIRS += injection_points gin
+ALWAYS_SUBDIRS += injection_points gin typcache
endif
ifeq ($(with_ssl),openssl)
diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build
index d8fe059d236..c829b619530 100644
--- a/src/test/modules/meson.build
+++ b/src/test/modules/meson.build
@@ -36,6 +36,7 @@ subdir('test_rls_hooks')
subdir('test_shm_mq')
subdir('test_slru')
subdir('test_tidstore')
+subdir('typcache')
subdir('unsafe_tests')
subdir('worker_spi')
subdir('xid_wraparound')
diff --git a/src/test/modules/typcache/.gitignore b/src/test/modules/typcache/.gitignore
new file mode 100644
index 00000000000..5dcb3ff9723
--- /dev/null
+++ b/src/test/modules/typcache/.gitignore
@@ -0,0 +1,4 @@
+# Generated subdirectories
+/log/
+/results/
+/tmp_check/
diff --git a/src/test/modules/typcache/Makefile b/src/test/modules/typcache/Makefile
new file mode 100644
index 00000000000..1f03de83890
--- /dev/null
+++ b/src/test/modules/typcache/Makefile
@@ -0,0 +1,28 @@
+# src/test/modules/typcache/Makefile
+
+EXTRA_INSTALL = src/test/modules/injection_points
+
+REGRESS = typcache_rel_type_cache
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = src/test/modules/typcache
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+
+# XXX: This test is conditional on enable_injection_points in the
+# parent Makefile, so we should never get here in the first place if
+# injection points are not enabled. But the buildfarm 'misc-check'
+# step doesn't pay attention to the if-condition in the parent
+# Makefile. To work around that, disable running the test here too.
+ifeq ($(enable_injection_points),yes)
+include $(top_srcdir)/contrib/contrib-global.mk
+else
+check:
+ @echo "injection points are disabled in this build"
+endif
+
+endif
diff --git a/src/test/modules/typcache/expected/typcache_rel_type_cache.out b/src/test/modules/typcache/expected/typcache_rel_type_cache.out
new file mode 100644
index 00000000000..b113e0bbd5d
--- /dev/null
+++ b/src/test/modules/typcache/expected/typcache_rel_type_cache.out
@@ -0,0 +1,34 @@
+--
+-- This test checks that lookup_type_cache() can correctly handle an
+-- interruption. We use the injection point to simulate an error but note
+-- that a similar situation could happen due to user query interruption.
+-- Despite the interruption, a map entry from the relation oid to type cache
+-- entry should be created. This is validated by subsequent modification of
+-- the table schema, then type casts which use new schema implying
+-- successful type cache invalidation by relation oid.
+--
+CREATE EXTENSION injection_points;
+CREATE TABLE t (i int);
+SELECT injection_points_attach('typecache-before-rel-type-cache-insert', 'error');
+ injection_points_attach
+-------------------------
+
+(1 row)
+
+SELECT '(1)'::t;
+ERROR: error triggered for injection point typecache-before-rel-type-cache-insert
+LINE 1: SELECT '(1)'::t;
+ ^
+SELECT injection_points_detach('typecache-before-rel-type-cache-insert');
+ injection_points_detach
+-------------------------
+
+(1 row)
+
+ALTER TABLE t ADD COLUMN j int;
+SELECT '(1,2)'::t;
+ t
+-------
+ (1,2)
+(1 row)
+
diff --git a/src/test/modules/typcache/meson.build b/src/test/modules/typcache/meson.build
new file mode 100644
index 00000000000..cb2e34c0d2b
--- /dev/null
+++ b/src/test/modules/typcache/meson.build
@@ -0,0 +1,16 @@
+# Copyright (c) 2022-2024, PostgreSQL Global Development Group
+
+if not get_option('injection_points')
+ subdir_done()
+endif
+
+tests += {
+ 'name': 'typcache',
+ 'sd': meson.current_source_dir(),
+ 'bd': meson.current_build_dir(),
+ 'regress': {
+ 'sql': [
+ 'typcache_rel_type_cache',
+ ],
+ },
+}
diff --git a/src/test/modules/typcache/sql/typcache_rel_type_cache.sql b/src/test/modules/typcache/sql/typcache_rel_type_cache.sql
new file mode 100644
index 00000000000..2c0a434d988
--- /dev/null
+++ b/src/test/modules/typcache/sql/typcache_rel_type_cache.sql
@@ -0,0 +1,18 @@
+--
+-- This test checks that lookup_type_cache() can correctly handle an
+-- interruption. We use the injection point to simulate an error but note
+-- that a similar situation could happen due to user query interruption.
+-- Despite the interruption, a map entry from the relation oid to type cache
+-- entry should be created. This is validated by subsequent modification of
+-- the table schema, then type casts which use new schema implying
+-- successful type cache invalidation by relation oid.
+--
+
+CREATE EXTENSION injection_points;
+
+CREATE TABLE t (i int);
+SELECT injection_points_attach('typecache-before-rel-type-cache-insert', 'error');
+SELECT '(1)'::t;
+SELECT injection_points_detach('typecache-before-rel-type-cache-insert');
+ALTER TABLE t ADD COLUMN j int;
+SELECT '(1,2)'::t;
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 57de1acff3a..9b3e7fd104b 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2378,6 +2378,7 @@ RelFileLocator
RelFileLocatorBackend
RelFileNumber
RelIdCacheEnt
+RelIdToTypeIdCacheEntry
RelInfo
RelInfoArr
RelMapFile
--
2.39.5 (Apple Git-154)
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2024-10-22 07:34 Alexander Korotkov <[email protected]>
parent: Alexander Korotkov <[email protected]>
0 siblings, 1 reply; 55+ messages in thread
From: Alexander Korotkov @ 2024-10-22 07:34 UTC (permalink / raw)
To: Dagfinn Ilmari Mannsåker <[email protected]>; +Cc: Andrei Lepikhov <[email protected]>; jian he <[email protected]>; Artur Zakirov <[email protected]>; Alexander Lakhin <[email protected]>; Pavel Borisov <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
On Mon, Oct 21, 2024 at 2:30 PM Alexander Korotkov <[email protected]> wrote:
>
> On Mon, Oct 21, 2024 at 1:16 PM Dagfinn Ilmari Mannsåker
> <[email protected]> wrote:
> > Alexander Korotkov <[email protected]> writes:
> >
> > > On Mon, Oct 21, 2024 at 8:40 AM Andrei Lepikhov <[email protected]> wrote:
> > >>
> > >> On 21/10/2024 06:32, Dagfinn Ilmari Mannsåker wrote:
> > >> > Alexander Korotkov <[email protected]> writes:
> > >> >
> > >> >> +static Oid *in_progress_list;
> > >> >> +static int in_progress_list_len;
> > >> >> +static int in_progress_list_maxlen;
> > >> >
> > >> > Is there any particular reason not to use pg_list.h for this?
> > >> Sure. The type cache lookup has to be as much optimal as possible.
> > >> Using an array and relating sequential access to it, we avoid memory
> > >> allocations and deallocations 99.9% of the time. Also, quick access to
> > >> the single element (which we will have in real life almost all of the
> > >> time) is much faster than employing list machinery.
> >
> > Lists are actually dynamically resized arrays these days (see commit
> > 1cff1b95ab6ddae32faa3efe0d95a820dbfdc164), not linked lists, so
> > accessing arbitrary elements is O(1), not O(n). Just like this patch,
> > the size is doubled (starting at 16) whenever array is full.
> >
> > > +1,
> > > List with zero elements has to be NIL. That means continuous
> > > allocations/deallocations.
> >
> > This however is a valid point (unless we keep a dummy zeroth element to
> > avoid it, which is even uglier than open-coding the array extension
> > logic), so objection withdrawn.
>
> OK, thank you!
>
> The attached revision fixes EXTRA_INSTALL in
> src/test/modules/typcache/Makefile. Spotted off-list by Arthur
> Zakirov.
I've re-checked that regression tests pass with
-DCLOBBER_CACHE_ALWAYS. Also did some grammar corrections for
comments and commit message. I'm going to push this if no objections.
------
Regards,
Alexander Korotkov
Supabase
Attachments:
[application/octet-stream] v18-0001-Update-header-comment-for-lookup_type_cache.patch (1.4K, ../../CAPpHfdtHjPuLPoLGAqeUnttxy-PooJNB4piiZraLXK2C8e8vwg@mail.gmail.com/2-v18-0001-Update-header-comment-for-lookup_type_cache.patch)
download | inline diff:
From ab098661c407355c07aacb7821221bfcbf10637b Mon Sep 17 00:00:00 2001
From: Alexander Korotkov <[email protected]>
Date: Tue, 22 Oct 2024 10:30:40 +0300
Subject: [PATCH v18 1/2] Update header comment for lookup_type_cache()
Describe the way we handle concurrent invalidation messages.
---
src/backend/utils/cache/typcache.c | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/src/backend/utils/cache/typcache.c b/src/backend/utils/cache/typcache.c
index 2ec136b7d30..11382547ec0 100644
--- a/src/backend/utils/cache/typcache.c
+++ b/src/backend/utils/cache/typcache.c
@@ -351,6 +351,15 @@ type_cache_syshash(const void *key, Size keysize)
* invalid. Note however that we may fail to find one or more of the
* values requested by 'flags'; the caller needs to check whether the fields
* are InvalidOid or not.
+ *
+ * Note that while filling TypeCacheEntry we might process concurrent
+ * invalidation messages, causing our not-yet-filled TypeCacheEntry to be
+ * invalidated. In this case, we typically only clear flags while values are
+ * still available for the caller. It's expected that the caller holds
+ * enough locks on type-depending objects that the values are still relevant.
+ * It's also important that the tupdesc is filled in the last for
+ * TYPTYPE_COMPOSITE. So, it can't get invalidated during the
+ * lookup_type_cache() call.
*/
TypeCacheEntry *
lookup_type_cache(Oid type_id, int flags)
--
2.39.5 (Apple Git-154)
[application/octet-stream] v18-0002-Avoid-looping-over-all-type-cache-entries-in-Typ.patch (24.7K, ../../CAPpHfdtHjPuLPoLGAqeUnttxy-PooJNB4piiZraLXK2C8e8vwg@mail.gmail.com/3-v18-0002-Avoid-looping-over-all-type-cache-entries-in-Typ.patch)
download | inline diff:
From aba5740d647bc16aa4bbe5cd82d59bd7b8df7ee3 Mon Sep 17 00:00:00 2001
From: Alexander Korotkov <[email protected]>
Date: Tue, 22 Oct 2024 10:30:46 +0300
Subject: [PATCH v18 2/2] Avoid looping over all type cache entries in
TypeCacheRelCallback()
Currently, when a single relcache entry gets invalidated,
TypeCacheRelCallback() has to loop over all type cache entries to find
appropriate typentry to invalidate. Unfortunately, using the syscache here
is impossible, because this callback could be called outside a transaction
and this makes impossible catalog lookups. This is why present commit
introduces RelIdToTypeIdCacheHash to map relation OID to its composite type
OID.
We are keeping RelIdToTypeIdCacheHash entry while corresponding type cache
entry have something to clean. Therefore, RelIdToTypeIdCacheHash shouldn't
get bloat in the case of temporary tables flood.
There are many places in lookup_type_cache() where syscache invalidation,
user interruption, or even error could occur. In order to handle this, we
keep an array of in-progress type cache entries. In the case of
lookup_type_cache() interruption this array is processed to keep
RelIdToTypeIdCacheHash in a consistent state.
Discussion: https://postgr.es/m/5812a6e5-68ae-4d84-9d85-b443176966a1%40sigaev.ru
Author: Teodor Sigaev
Reviewed-by: Aleksander Alekseev, Tom Lane, Michael Paquier, Roman Zharkov
Reviewed-by: Andrei Lepikhov, Pavel Borisov, Jian He, Alexander Lakhin
Reviewed-by: Artur Zakirov
---
src/backend/access/transam/xact.c | 10 +
src/backend/utils/cache/typcache.c | 359 +++++++++++++++---
src/include/utils/typcache.h | 4 +
src/test/modules/Makefile | 4 +-
src/test/modules/meson.build | 1 +
src/test/modules/typcache/.gitignore | 4 +
src/test/modules/typcache/Makefile | 28 ++
.../expected/typcache_rel_type_cache.out | 34 ++
src/test/modules/typcache/meson.build | 16 +
.../typcache/sql/typcache_rel_type_cache.sql | 18 +
src/tools/pgindent/typedefs.list | 1 +
11 files changed, 433 insertions(+), 46 deletions(-)
create mode 100644 src/test/modules/typcache/.gitignore
create mode 100644 src/test/modules/typcache/Makefile
create mode 100644 src/test/modules/typcache/expected/typcache_rel_type_cache.out
create mode 100644 src/test/modules/typcache/meson.build
create mode 100644 src/test/modules/typcache/sql/typcache_rel_type_cache.sql
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 87700c7c5c7..b0b05e28790 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -70,6 +70,7 @@
#include "utils/snapmgr.h"
#include "utils/timeout.h"
#include "utils/timestamp.h"
+#include "utils/typcache.h"
/*
* User-tweakable parameters
@@ -2407,6 +2408,9 @@ CommitTransaction(void)
/* Clean up the relation cache */
AtEOXact_RelationCache(true);
+ /* Clean up the type cache */
+ AtEOXact_TypeCache();
+
/*
* Make catalog changes visible to all backends. This has to happen after
* relcache references are dropped (see comments for
@@ -2709,6 +2713,9 @@ PrepareTransaction(void)
/* Clean up the relation cache */
AtEOXact_RelationCache(true);
+ /* Clean up the type cache */
+ AtEOXact_TypeCache();
+
/* notify doesn't need a postprepare call */
PostPrepare_PgStat();
@@ -2951,6 +2958,7 @@ AbortTransaction(void)
false, true);
AtEOXact_Buffers(false);
AtEOXact_RelationCache(false);
+ AtEOXact_TypeCache();
AtEOXact_Inval(false);
AtEOXact_MultiXact();
ResourceOwnerRelease(TopTransactionResourceOwner,
@@ -5153,6 +5161,7 @@ CommitSubTransaction(void)
true, false);
AtEOSubXact_RelationCache(true, s->subTransactionId,
s->parent->subTransactionId);
+ AtEOSubXact_TypeCache();
AtEOSubXact_Inval(true);
AtSubCommit_smgr();
@@ -5328,6 +5337,7 @@ AbortSubTransaction(void)
AtEOSubXact_RelationCache(false, s->subTransactionId,
s->parent->subTransactionId);
+ AtEOSubXact_TypeCache();
AtEOSubXact_Inval(false);
ResourceOwnerRelease(s->curTransactionOwner,
RESOURCE_RELEASE_LOCKS,
diff --git a/src/backend/utils/cache/typcache.c b/src/backend/utils/cache/typcache.c
index 11382547ec0..2037e2f1c16 100644
--- a/src/backend/utils/cache/typcache.c
+++ b/src/backend/utils/cache/typcache.c
@@ -66,6 +66,7 @@
#include "utils/builtins.h"
#include "utils/catcache.h"
#include "utils/fmgroids.h"
+#include "utils/injection_point.h"
#include "utils/inval.h"
#include "utils/lsyscache.h"
#include "utils/memutils.h"
@@ -77,6 +78,20 @@
/* The main type cache hashtable searched by lookup_type_cache */
static HTAB *TypeCacheHash = NULL;
+/*
+ * The mapping of relation's OID to the corresponding composite type OID.
+ * We're keeping the map entry when the corresponding typentry has something
+ * to clear i.e it has either TCFLAGS_HAVE_PG_TYPE_DATA, or
+ * TCFLAGS_OPERATOR_FLAGS, or tupdesc.
+ */
+static HTAB *RelIdToTypeIdCacheHash = NULL;
+
+typedef struct RelIdToTypeIdCacheEntry
+{
+ Oid relid; /* OID of the relation */
+ Oid composite_typid; /* OID of the relation's composite type */
+} RelIdToTypeIdCacheEntry;
+
/* List of type cache entries for domain types */
static TypeCacheEntry *firstDomainTypeEntry = NULL;
@@ -208,6 +223,10 @@ typedef struct SharedTypmodTableEntry
dsa_pointer shared_tupdesc;
} SharedTypmodTableEntry;
+static Oid *in_progress_list;
+static int in_progress_list_len;
+static int in_progress_list_maxlen;
+
/*
* A comparator function for SharedRecordTableKey.
*/
@@ -329,6 +348,8 @@ static void shared_record_typmod_registry_detach(dsm_segment *segment,
static TupleDesc find_or_make_matching_shared_tupledesc(TupleDesc tupdesc);
static dsa_pointer share_tupledesc(dsa_area *area, TupleDesc tupdesc,
uint32 typmod);
+static void insert_rel_type_cache_if_needed(TypeCacheEntry *typentry);
+static void delete_rel_type_cache_if_needed(TypeCacheEntry *typentry);
/*
@@ -366,11 +387,13 @@ lookup_type_cache(Oid type_id, int flags)
{
TypeCacheEntry *typentry;
bool found;
+ int in_progress_offset;
if (TypeCacheHash == NULL)
{
/* First time through: initialize the hash table */
HASHCTL ctl;
+ int allocsize;
ctl.keysize = sizeof(Oid);
ctl.entrysize = sizeof(TypeCacheEntry);
@@ -385,6 +408,13 @@ lookup_type_cache(Oid type_id, int flags)
TypeCacheHash = hash_create("Type information cache", 64,
&ctl, HASH_ELEM | HASH_FUNCTION);
+ Assert(RelIdToTypeIdCacheHash == NULL);
+
+ ctl.keysize = sizeof(Oid);
+ ctl.entrysize = sizeof(RelIdToTypeIdCacheEntry);
+ RelIdToTypeIdCacheHash = hash_create("Map from relid to OID of cached composite type", 64,
+ &ctl, HASH_ELEM | HASH_BLOBS);
+
/* Also set up callbacks for SI invalidations */
CacheRegisterRelcacheCallback(TypeCacheRelCallback, (Datum) 0);
CacheRegisterSyscacheCallback(TYPEOID, TypeCacheTypCallback, (Datum) 0);
@@ -394,8 +424,32 @@ lookup_type_cache(Oid type_id, int flags)
/* Also make sure CacheMemoryContext exists */
if (!CacheMemoryContext)
CreateCacheMemoryContext();
+
+ /*
+ * reserve enough in_progress_list slots for many cases
+ */
+ allocsize = 4;
+ in_progress_list =
+ MemoryContextAlloc(CacheMemoryContext,
+ allocsize * sizeof(*in_progress_list));
+ in_progress_list_maxlen = allocsize;
}
+ Assert(TypeCacheHash != NULL && RelIdToTypeIdCacheHash != NULL);
+
+ /* Register to catch invalidation messages */
+ if (in_progress_list_len >= in_progress_list_maxlen)
+ {
+ int allocsize;
+
+ allocsize = in_progress_list_maxlen * 2;
+ in_progress_list = repalloc(in_progress_list,
+ allocsize * sizeof(*in_progress_list));
+ in_progress_list_maxlen = allocsize;
+ }
+ in_progress_offset = in_progress_list_len++;
+ in_progress_list[in_progress_offset] = type_id;
+
/* Try to look up an existing entry */
typentry = (TypeCacheEntry *) hash_search(TypeCacheHash,
&type_id,
@@ -896,6 +950,13 @@ lookup_type_cache(Oid type_id, int flags)
load_domaintype_info(typentry);
}
+ INJECTION_POINT("typecache-before-rel-type-cache-insert");
+
+ Assert(in_progress_offset + 1 == in_progress_list_len);
+ in_progress_list_len--;
+
+ insert_rel_type_cache_if_needed(typentry);
+
return typentry;
}
@@ -2290,6 +2351,53 @@ SharedRecordTypmodRegistryAttach(SharedRecordTypmodRegistry *registry)
CurrentSession->shared_typmod_table = typmod_table;
}
+/*
+ * InvalidateCompositeTypeCacheEntry
+ * Invalidate particular TypeCacheEntry on Relcache inval callback
+ *
+ * Delete the cached tuple descriptor (if any) for the given composite
+ * type, and reset whatever info we have cached about the composite type's
+ * comparability.
+ */
+static void
+InvalidateCompositeTypeCacheEntry(TypeCacheEntry *typentry)
+{
+ bool hadTupDescOrOpclass;
+
+ Assert(typentry->typtype == TYPTYPE_COMPOSITE &&
+ OidIsValid(typentry->typrelid));
+
+ hadTupDescOrOpclass = (typentry->tupDesc != NULL) ||
+ (typentry->flags & TCFLAGS_OPERATOR_FLAGS);
+
+ /* Delete tupdesc if we have it */
+ if (typentry->tupDesc != NULL)
+ {
+ /*
+ * Release our refcount and free the tupdesc if none remain. We can't
+ * use DecrTupleDescRefCount here because this reference is not logged
+ * by the current resource owner.
+ */
+ Assert(typentry->tupDesc->tdrefcount > 0);
+ if (--typentry->tupDesc->tdrefcount == 0)
+ FreeTupleDesc(typentry->tupDesc);
+ typentry->tupDesc = NULL;
+
+ /*
+ * Also clear tupDesc_identifier, so that anyone watching it will
+ * realize that the tupdesc has changed.
+ */
+ typentry->tupDesc_identifier = 0;
+ }
+
+ /* Reset equality/comparison/hashing validity information */
+ typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
+
+ /* Call delete_rel_type_cache() if we actually cleared something */
+ if (hadTupDescOrOpclass)
+ delete_rel_type_cache_if_needed(typentry);
+}
+
/*
* TypeCacheRelCallback
* Relcache inval callback function
@@ -2299,63 +2407,55 @@ SharedRecordTypmodRegistryAttach(SharedRecordTypmodRegistry *registry)
* whatever info we have cached about the composite type's comparability.
*
* This is called when a relcache invalidation event occurs for the given
- * relid. We must scan the whole typcache hash since we don't know the
- * type OID corresponding to the relid. We could do a direct search if this
- * were a syscache-flush callback on pg_type, but then we would need all
- * ALTER-TABLE-like commands that could modify a rowtype to issue syscache
- * invals against the rel's pg_type OID. The extra SI signaling could very
- * well cost more than we'd save, since in most usages there are not very
- * many entries in a backend's typcache. The risk of bugs-of-omission seems
- * high, too.
- *
- * Another possibility, with only localized impact, is to maintain a second
- * hashtable that indexes composite-type typcache entries by their typrelid.
- * But it's still not clear it's worth the trouble.
+ * relid. We can't use syscache to find a type corresponding to the given
+ * relation because the code can be called outside of transaction. Thus, we
+ * use the RelIdToTypeIdCacheHash map to locate appropriate typcache entry.
*/
static void
TypeCacheRelCallback(Datum arg, Oid relid)
{
- HASH_SEQ_STATUS status;
TypeCacheEntry *typentry;
- /* TypeCacheHash must exist, else this callback wouldn't be registered */
- hash_seq_init(&status, TypeCacheHash);
- while ((typentry = (TypeCacheEntry *) hash_seq_search(&status)) != NULL)
+ /*
+ * RelIdToTypeIdCacheHash and TypeCacheHash should exist, otherwise this
+ * callback wouldn't be registered
+ */
+ if (OidIsValid(relid))
{
- if (typentry->typtype == TYPTYPE_COMPOSITE)
+ RelIdToTypeIdCacheEntry *relentry;
+
+ /*
+ * Find an RelIdToTypeIdCacheHash entry, which should exist as soon as
+ * corresponding typcache entry has something to clean.
+ */
+ relentry = (RelIdToTypeIdCacheEntry *) hash_search(RelIdToTypeIdCacheHash,
+ &relid,
+ HASH_FIND, NULL);
+
+ if (relentry != NULL)
{
- /* Skip if no match, unless we're zapping all composite types */
- if (relid != typentry->typrelid && relid != InvalidOid)
- continue;
+ typentry = (TypeCacheEntry *) hash_search(TypeCacheHash,
+ &relentry->composite_typid,
+ HASH_FIND, NULL);
- /* Delete tupdesc if we have it */
- if (typentry->tupDesc != NULL)
+ if (typentry != NULL)
{
- /*
- * Release our refcount, and free the tupdesc if none remain.
- * (Can't use DecrTupleDescRefCount because this reference is
- * not logged in current resource owner.)
- */
- Assert(typentry->tupDesc->tdrefcount > 0);
- if (--typentry->tupDesc->tdrefcount == 0)
- FreeTupleDesc(typentry->tupDesc);
- typentry->tupDesc = NULL;
+ Assert(typentry->typtype == TYPTYPE_COMPOSITE);
+ Assert(relid == typentry->typrelid);
- /*
- * Also clear tupDesc_identifier, so that anything watching
- * that will realize that the tupdesc has possibly changed.
- * (Alternatively, we could specify that to detect possible
- * tupdesc change, one must check for tupDesc != NULL as well
- * as tupDesc_identifier being the same as what was previously
- * seen. That seems error-prone.)
- */
- typentry->tupDesc_identifier = 0;
+ InvalidateCompositeTypeCacheEntry(typentry);
}
-
- /* Reset equality/comparison/hashing validity information */
- typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
}
- else if (typentry->typtype == TYPTYPE_DOMAIN)
+
+ /*
+ * Visit all the domain types sequentially. Typically, this shouldn't
+ * affect performance since domain types are less tended to bloat.
+ * Domain types are created manually, unlike composite types which are
+ * automatically created for every temporary table.
+ */
+ for (typentry = firstDomainTypeEntry;
+ typentry != NULL;
+ typentry = typentry->nextDomain)
{
/*
* If it's domain over composite, reset flags. (We don't bother
@@ -2367,6 +2467,36 @@ TypeCacheRelCallback(Datum arg, Oid relid)
typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
}
}
+ else
+ {
+ HASH_SEQ_STATUS status;
+
+ /*
+ * Relid is invalid. By convention, we need to reset all composite
+ * types in cache. Also, we should reset flags for domain types, and
+ * we loop over all entries in hash, so, do it in a single scan.
+ */
+ hash_seq_init(&status, TypeCacheHash);
+ while ((typentry = (TypeCacheEntry *) hash_seq_search(&status)) != NULL)
+ {
+ if (typentry->typtype == TYPTYPE_COMPOSITE)
+ {
+ InvalidateCompositeTypeCacheEntry(typentry);
+ }
+ else if (typentry->typtype == TYPTYPE_DOMAIN)
+ {
+ /*
+ * If it's domain over composite, reset flags. (We don't
+ * bother trying to determine whether the specific base type
+ * needs a reset.) Note that if we haven't determined whether
+ * the base type is composite, we don't need to reset
+ * anything.
+ */
+ if (typentry->flags & TCFLAGS_DOMAIN_BASE_IS_COMPOSITE)
+ typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
+ }
+ }
+ }
}
/*
@@ -2397,6 +2527,8 @@ TypeCacheTypCallback(Datum arg, int cacheid, uint32 hashvalue)
while ((typentry = (TypeCacheEntry *) hash_seq_search(&status)) != NULL)
{
+ bool hadPgTypeData = (typentry->flags & TCFLAGS_HAVE_PG_TYPE_DATA);
+
Assert(hashvalue == 0 || typentry->type_id_hash == hashvalue);
/*
@@ -2406,6 +2538,13 @@ TypeCacheTypCallback(Datum arg, int cacheid, uint32 hashvalue)
*/
typentry->flags &= ~(TCFLAGS_HAVE_PG_TYPE_DATA |
TCFLAGS_CHECKED_DOMAIN_CONSTRAINTS);
+
+ /*
+ * Call delete_rel_type_cache() if we cleaned
+ * TCFLAGS_HAVE_PG_TYPE_DATA flag previously.
+ */
+ if (hadPgTypeData)
+ delete_rel_type_cache_if_needed(typentry);
}
}
@@ -2914,3 +3053,135 @@ shared_record_typmod_registry_detach(dsm_segment *segment, Datum datum)
}
CurrentSession->shared_typmod_registry = NULL;
}
+
+/*
+ * Insert RelIdToTypeIdCacheHash entry if needed.
+ */
+static void
+insert_rel_type_cache_if_needed(TypeCacheEntry *typentry)
+{
+ /* Immediately quit for non-composite types */
+ if (typentry->typtype != TYPTYPE_COMPOSITE)
+ return;
+
+ /* typrelid should be given for composite types */
+ Assert(OidIsValid(typentry->typrelid));
+
+ /*
+ * Insert a RelIdToTypeIdCacheHash entry if the typentry have any
+ * information indicating it should be here.
+ */
+ if ((typentry->flags & TCFLAGS_HAVE_PG_TYPE_DATA) ||
+ (typentry->flags & TCFLAGS_OPERATOR_FLAGS) ||
+ typentry->tupDesc != NULL)
+ {
+ RelIdToTypeIdCacheEntry *relentry;
+ bool found;
+
+ relentry = (RelIdToTypeIdCacheEntry *) hash_search(RelIdToTypeIdCacheHash,
+ &typentry->typrelid,
+ HASH_ENTER, &found);
+ relentry->relid = typentry->typrelid;
+ relentry->composite_typid = typentry->type_id;
+ }
+}
+
+/*
+ * Delete entry RelIdToTypeIdCacheHash if needed after resetting of the
+ * TCFLAGS_HAVE_PG_TYPE_DATA flag, or any of TCFLAGS_OPERATOR_FLAGS,
+ * or tupDesc.
+ */
+static void
+delete_rel_type_cache_if_needed(TypeCacheEntry *typentry)
+{
+#ifdef USE_ASSERT_CHECKING
+ int i;
+ bool is_in_progress = false;
+
+ for (i = 0; i < in_progress_list_len; i++)
+ {
+ if (in_progress_list[i] == typentry->type_id)
+ {
+ is_in_progress = true;
+ break;
+ }
+ }
+#endif
+
+ /* Immediately quit for non-composite types */
+ if (typentry->typtype != TYPTYPE_COMPOSITE)
+ return;
+
+ /* typrelid should be given for composite types */
+ Assert(OidIsValid(typentry->typrelid));
+
+ /*
+ * Delete a RelIdToTypeIdCacheHash entry if the typentry doesn't have any
+ * information indicating entry should be still there.
+ */
+ if (!(typentry->flags & TCFLAGS_HAVE_PG_TYPE_DATA) &&
+ !(typentry->flags & TCFLAGS_OPERATOR_FLAGS) &&
+ typentry->tupDesc == NULL)
+ {
+ bool found;
+
+ (void) hash_search(RelIdToTypeIdCacheHash,
+ &typentry->typrelid,
+ HASH_REMOVE, &found);
+ Assert(found || is_in_progress);
+ }
+ else
+ {
+#ifdef USE_ASSERT_CHECKING
+ /*
+ * In assert-enabled builds otherwise check for RelIdToTypeIdCacheHash
+ * entry if it should exist.
+ */
+ bool found;
+
+ if (!is_in_progress)
+ {
+ (void) hash_search(RelIdToTypeIdCacheHash,
+ &typentry->typrelid,
+ HASH_FIND, &found);
+ Assert(found);
+ }
+#endif
+ }
+}
+
+/*
+ * Add possibly missing RelIdToTypeId entries related to TypeCacheHash
+ * entries, marked as in-progress by lookup_type_cache(). It may happen
+ * in case of an error or interruption during the lookup_type_cache() call.
+ */
+static void
+finalize_in_progress_typentries(void)
+{
+ int i;
+
+ for (i = 0; i < in_progress_list_len; i++)
+ {
+ TypeCacheEntry *typentry;
+
+ typentry = (TypeCacheEntry *) hash_search(TypeCacheHash,
+ &in_progress_list[i],
+ HASH_FIND, NULL);
+ if (typentry)
+ insert_rel_type_cache_if_needed(typentry);
+ }
+
+ in_progress_list_len = 0;
+}
+
+void
+AtEOXact_TypeCache(void)
+{
+ finalize_in_progress_typentries();
+}
+
+void
+AtEOSubXact_TypeCache(void)
+{
+ finalize_in_progress_typentries();
+}
diff --git a/src/include/utils/typcache.h b/src/include/utils/typcache.h
index f506cc4aa35..f3d73ecee3a 100644
--- a/src/include/utils/typcache.h
+++ b/src/include/utils/typcache.h
@@ -207,4 +207,8 @@ extern void SharedRecordTypmodRegistryInit(SharedRecordTypmodRegistry *,
extern void SharedRecordTypmodRegistryAttach(SharedRecordTypmodRegistry *);
+extern void AtEOXact_TypeCache(void);
+
+extern void AtEOSubXact_TypeCache(void);
+
#endif /* TYPCACHE_H */
diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile
index 256799f520a..c0d3cf0e14b 100644
--- a/src/test/modules/Makefile
+++ b/src/test/modules/Makefile
@@ -43,9 +43,9 @@ SUBDIRS = \
ifeq ($(enable_injection_points),yes)
-SUBDIRS += injection_points gin
+SUBDIRS += injection_points gin typcache
else
-ALWAYS_SUBDIRS += injection_points gin
+ALWAYS_SUBDIRS += injection_points gin typcache
endif
ifeq ($(with_ssl),openssl)
diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build
index d8fe059d236..c829b619530 100644
--- a/src/test/modules/meson.build
+++ b/src/test/modules/meson.build
@@ -36,6 +36,7 @@ subdir('test_rls_hooks')
subdir('test_shm_mq')
subdir('test_slru')
subdir('test_tidstore')
+subdir('typcache')
subdir('unsafe_tests')
subdir('worker_spi')
subdir('xid_wraparound')
diff --git a/src/test/modules/typcache/.gitignore b/src/test/modules/typcache/.gitignore
new file mode 100644
index 00000000000..5dcb3ff9723
--- /dev/null
+++ b/src/test/modules/typcache/.gitignore
@@ -0,0 +1,4 @@
+# Generated subdirectories
+/log/
+/results/
+/tmp_check/
diff --git a/src/test/modules/typcache/Makefile b/src/test/modules/typcache/Makefile
new file mode 100644
index 00000000000..1f03de83890
--- /dev/null
+++ b/src/test/modules/typcache/Makefile
@@ -0,0 +1,28 @@
+# src/test/modules/typcache/Makefile
+
+EXTRA_INSTALL = src/test/modules/injection_points
+
+REGRESS = typcache_rel_type_cache
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = src/test/modules/typcache
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+
+# XXX: This test is conditional on enable_injection_points in the
+# parent Makefile, so we should never get here in the first place if
+# injection points are not enabled. But the buildfarm 'misc-check'
+# step doesn't pay attention to the if-condition in the parent
+# Makefile. To work around that, disable running the test here too.
+ifeq ($(enable_injection_points),yes)
+include $(top_srcdir)/contrib/contrib-global.mk
+else
+check:
+ @echo "injection points are disabled in this build"
+endif
+
+endif
diff --git a/src/test/modules/typcache/expected/typcache_rel_type_cache.out b/src/test/modules/typcache/expected/typcache_rel_type_cache.out
new file mode 100644
index 00000000000..b113e0bbd5d
--- /dev/null
+++ b/src/test/modules/typcache/expected/typcache_rel_type_cache.out
@@ -0,0 +1,34 @@
+--
+-- This test checks that lookup_type_cache() can correctly handle an
+-- interruption. We use the injection point to simulate an error but note
+-- that a similar situation could happen due to user query interruption.
+-- Despite the interruption, a map entry from the relation oid to type cache
+-- entry should be created. This is validated by subsequent modification of
+-- the table schema, then type casts which use new schema implying
+-- successful type cache invalidation by relation oid.
+--
+CREATE EXTENSION injection_points;
+CREATE TABLE t (i int);
+SELECT injection_points_attach('typecache-before-rel-type-cache-insert', 'error');
+ injection_points_attach
+-------------------------
+
+(1 row)
+
+SELECT '(1)'::t;
+ERROR: error triggered for injection point typecache-before-rel-type-cache-insert
+LINE 1: SELECT '(1)'::t;
+ ^
+SELECT injection_points_detach('typecache-before-rel-type-cache-insert');
+ injection_points_detach
+-------------------------
+
+(1 row)
+
+ALTER TABLE t ADD COLUMN j int;
+SELECT '(1,2)'::t;
+ t
+-------
+ (1,2)
+(1 row)
+
diff --git a/src/test/modules/typcache/meson.build b/src/test/modules/typcache/meson.build
new file mode 100644
index 00000000000..cb2e34c0d2b
--- /dev/null
+++ b/src/test/modules/typcache/meson.build
@@ -0,0 +1,16 @@
+# Copyright (c) 2022-2024, PostgreSQL Global Development Group
+
+if not get_option('injection_points')
+ subdir_done()
+endif
+
+tests += {
+ 'name': 'typcache',
+ 'sd': meson.current_source_dir(),
+ 'bd': meson.current_build_dir(),
+ 'regress': {
+ 'sql': [
+ 'typcache_rel_type_cache',
+ ],
+ },
+}
diff --git a/src/test/modules/typcache/sql/typcache_rel_type_cache.sql b/src/test/modules/typcache/sql/typcache_rel_type_cache.sql
new file mode 100644
index 00000000000..2c0a434d988
--- /dev/null
+++ b/src/test/modules/typcache/sql/typcache_rel_type_cache.sql
@@ -0,0 +1,18 @@
+--
+-- This test checks that lookup_type_cache() can correctly handle an
+-- interruption. We use the injection point to simulate an error but note
+-- that a similar situation could happen due to user query interruption.
+-- Despite the interruption, a map entry from the relation oid to type cache
+-- entry should be created. This is validated by subsequent modification of
+-- the table schema, then type casts which use new schema implying
+-- successful type cache invalidation by relation oid.
+--
+
+CREATE EXTENSION injection_points;
+
+CREATE TABLE t (i int);
+SELECT injection_points_attach('typecache-before-rel-type-cache-insert', 'error');
+SELECT '(1)'::t;
+SELECT injection_points_detach('typecache-before-rel-type-cache-insert');
+ALTER TABLE t ADD COLUMN j int;
+SELECT '(1,2)'::t;
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 57de1acff3a..9b3e7fd104b 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2378,6 +2378,7 @@ RelFileLocator
RelFileLocatorBackend
RelFileNumber
RelIdCacheEnt
+RelIdToTypeIdCacheEntry
RelInfo
RelInfoArr
RelMapFile
--
2.39.5 (Apple Git-154)
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2024-10-22 15:09 Pavel Borisov <[email protected]>
parent: Alexander Korotkov <[email protected]>
0 siblings, 1 reply; 55+ messages in thread
From: Pavel Borisov @ 2024-10-22 15:09 UTC (permalink / raw)
To: Alexander Korotkov <[email protected]>; +Cc: Dagfinn Ilmari Mannsåker <[email protected]>; Andrei Lepikhov <[email protected]>; jian he <[email protected]>; Artur Zakirov <[email protected]>; Alexander Lakhin <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
Hi, Alexander!
On Tue, 22 Oct 2024 at 11:34, Alexander Korotkov <[email protected]>
wrote:
> On Mon, Oct 21, 2024 at 2:30 PM Alexander Korotkov <[email protected]>
> wrote:
> >
> > On Mon, Oct 21, 2024 at 1:16 PM Dagfinn Ilmari Mannsåker
> > <[email protected]> wrote:
> > > Alexander Korotkov <[email protected]> writes:
> > >
> > > > On Mon, Oct 21, 2024 at 8:40 AM Andrei Lepikhov <[email protected]>
> wrote:
> > > >>
> > > >> On 21/10/2024 06:32, Dagfinn Ilmari Mannsåker wrote:
> > > >> > Alexander Korotkov <[email protected]> writes:
> > > >> >
> > > >> >> +static Oid *in_progress_list;
> > > >> >> +static int in_progress_list_len;
> > > >> >> +static int in_progress_list_maxlen;
> > > >> >
> > > >> > Is there any particular reason not to use pg_list.h for this?
> > > >> Sure. The type cache lookup has to be as much optimal as possible.
> > > >> Using an array and relating sequential access to it, we avoid memory
> > > >> allocations and deallocations 99.9% of the time. Also, quick access
> to
> > > >> the single element (which we will have in real life almost all of
> the
> > > >> time) is much faster than employing list machinery.
> > >
> > > Lists are actually dynamically resized arrays these days (see commit
> > > 1cff1b95ab6ddae32faa3efe0d95a820dbfdc164), not linked lists, so
> > > accessing arbitrary elements is O(1), not O(n). Just like this patch,
> > > the size is doubled (starting at 16) whenever array is full.
> > >
> > > > +1,
> > > > List with zero elements has to be NIL. That means continuous
> > > > allocations/deallocations.
> > >
> > > This however is a valid point (unless we keep a dummy zeroth element to
> > > avoid it, which is even uglier than open-coding the array extension
> > > logic), so objection withdrawn.
> >
> > OK, thank you!
> >
> > The attached revision fixes EXTRA_INSTALL in
> > src/test/modules/typcache/Makefile. Spotted off-list by Arthur
> > Zakirov.
>
> I've re-checked that regression tests pass with
> -DCLOBBER_CACHE_ALWAYS. Also did some grammar corrections for
> comments and commit message. I'm going to push this if no objections.
>
Thank you for working on this patch!
Looked through the patchset once more.
Patch 0001 (minor): "in the last" -> "after everything else" or "after
other TypeCacheEntry contents"
Patch 0002 looks ready to me.
Regards,
Pavel Borisov
Supabase
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2024-10-22 17:33 Alexander Korotkov <[email protected]>
parent: Pavel Borisov <[email protected]>
0 siblings, 2 replies; 55+ messages in thread
From: Alexander Korotkov @ 2024-10-22 17:33 UTC (permalink / raw)
To: Pavel Borisov <[email protected]>; +Cc: Dagfinn Ilmari Mannsåker <[email protected]>; Andrei Lepikhov <[email protected]>; jian he <[email protected]>; Artur Zakirov <[email protected]>; Alexander Lakhin <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
On Tue, Oct 22, 2024 at 6:10 PM Pavel Borisov <[email protected]> wrote:
> On Tue, 22 Oct 2024 at 11:34, Alexander Korotkov <[email protected]> wrote:
>>
>> On Mon, Oct 21, 2024 at 2:30 PM Alexander Korotkov <[email protected]> wrote:
>> >
>> > On Mon, Oct 21, 2024 at 1:16 PM Dagfinn Ilmari Mannsåker
>> > <[email protected]> wrote:
>> > > Alexander Korotkov <[email protected]> writes:
>> > >
>> > > > On Mon, Oct 21, 2024 at 8:40 AM Andrei Lepikhov <[email protected]> wrote:
>> > > >>
>> > > >> On 21/10/2024 06:32, Dagfinn Ilmari Mannsåker wrote:
>> > > >> > Alexander Korotkov <[email protected]> writes:
>> > > >> >
>> > > >> >> +static Oid *in_progress_list;
>> > > >> >> +static int in_progress_list_len;
>> > > >> >> +static int in_progress_list_maxlen;
>> > > >> >
>> > > >> > Is there any particular reason not to use pg_list.h for this?
>> > > >> Sure. The type cache lookup has to be as much optimal as possible.
>> > > >> Using an array and relating sequential access to it, we avoid memory
>> > > >> allocations and deallocations 99.9% of the time. Also, quick access to
>> > > >> the single element (which we will have in real life almost all of the
>> > > >> time) is much faster than employing list machinery.
>> > >
>> > > Lists are actually dynamically resized arrays these days (see commit
>> > > 1cff1b95ab6ddae32faa3efe0d95a820dbfdc164), not linked lists, so
>> > > accessing arbitrary elements is O(1), not O(n). Just like this patch,
>> > > the size is doubled (starting at 16) whenever array is full.
>> > >
>> > > > +1,
>> > > > List with zero elements has to be NIL. That means continuous
>> > > > allocations/deallocations.
>> > >
>> > > This however is a valid point (unless we keep a dummy zeroth element to
>> > > avoid it, which is even uglier than open-coding the array extension
>> > > logic), so objection withdrawn.
>> >
>> > OK, thank you!
>> >
>> > The attached revision fixes EXTRA_INSTALL in
>> > src/test/modules/typcache/Makefile. Spotted off-list by Arthur
>> > Zakirov.
>>
>> I've re-checked that regression tests pass with
>> -DCLOBBER_CACHE_ALWAYS. Also did some grammar corrections for
>> comments and commit message. I'm going to push this if no objections.
>
> Thank you for working on this patch!
> Looked through the patchset once more.
>
> Patch 0001 (minor): "in the last" -> "after everything else" or "after other TypeCacheEntry contents"
>
> Patch 0002 looks ready to me.
Thank you, Pavel! 0001 revised according to your suggestion.
------
Regards,
Alexander Korotkov
Supabase
Attachments:
[application/octet-stream] v19-0001-Update-header-comment-for-lookup_type_cache.patch (1.6K, ../../CAPpHfdv=WVCbzdpoTxvXzUZZEqu_pDk2YZBnEGpUCrzuHxPN6w@mail.gmail.com/2-v19-0001-Update-header-comment-for-lookup_type_cache.patch)
download | inline diff:
From 5f9c300e039854949cbd1337200b227018426c05 Mon Sep 17 00:00:00 2001
From: Alexander Korotkov <[email protected]>
Date: Tue, 22 Oct 2024 10:30:40 +0300
Subject: [PATCH v19 1/2] Update header comment for lookup_type_cache()
Describe the way we handle concurrent invalidation messages.
Discussion: https://postgr.es/m/CAPpHfdsQhwUrnB3of862j9RgHoJM--eRbifvBMvtQxpC57dxCA%40mail.gmail.com
Reviewed-by: Andrei Lepikhov, Artur Zakirov, Pavel Borisov
---
src/backend/utils/cache/typcache.c | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/src/backend/utils/cache/typcache.c b/src/backend/utils/cache/typcache.c
index 2ec136b7d30..f142624ad2e 100644
--- a/src/backend/utils/cache/typcache.c
+++ b/src/backend/utils/cache/typcache.c
@@ -351,6 +351,15 @@ type_cache_syshash(const void *key, Size keysize)
* invalid. Note however that we may fail to find one or more of the
* values requested by 'flags'; the caller needs to check whether the fields
* are InvalidOid or not.
+ *
+ * Note that while filling TypeCacheEntry we might process concurrent
+ * invalidation messages, causing our not-yet-filled TypeCacheEntry to be
+ * invalidated. In this case, we typically only clear flags while values are
+ * still available for the caller. It's expected that the caller holds
+ * enough locks on type-depending objects that the values are still relevant.
+ * It's also important that the tupdesc is filled after all other
+ * TypeCacheEntry items for TYPTYPE_COMPOSITE. So, tupdesc can't get
+ * invalidated during the lookup_type_cache() call.
*/
TypeCacheEntry *
lookup_type_cache(Oid type_id, int flags)
--
2.39.5 (Apple Git-154)
[application/octet-stream] v19-0002-Avoid-looping-over-all-type-cache-entries-in-Typ.patch (24.7K, ../../CAPpHfdv=WVCbzdpoTxvXzUZZEqu_pDk2YZBnEGpUCrzuHxPN6w@mail.gmail.com/3-v19-0002-Avoid-looping-over-all-type-cache-entries-in-Typ.patch)
download | inline diff:
From af2a07a7cc52f3335c939e0ed84bcca70a50b885 Mon Sep 17 00:00:00 2001
From: Alexander Korotkov <[email protected]>
Date: Tue, 22 Oct 2024 10:30:46 +0300
Subject: [PATCH v19 2/2] Avoid looping over all type cache entries in
TypeCacheRelCallback()
Currently, when a single relcache entry gets invalidated,
TypeCacheRelCallback() has to loop over all type cache entries to find
appropriate typentry to invalidate. Unfortunately, using the syscache here
is impossible, because this callback could be called outside a transaction
and this makes impossible catalog lookups. This is why present commit
introduces RelIdToTypeIdCacheHash to map relation OID to its composite type
OID.
We are keeping RelIdToTypeIdCacheHash entry while corresponding type cache
entry have something to clean. Therefore, RelIdToTypeIdCacheHash shouldn't
get bloat in the case of temporary tables flood.
There are many places in lookup_type_cache() where syscache invalidation,
user interruption, or even error could occur. In order to handle this, we
keep an array of in-progress type cache entries. In the case of
lookup_type_cache() interruption this array is processed to keep
RelIdToTypeIdCacheHash in a consistent state.
Discussion: https://postgr.es/m/5812a6e5-68ae-4d84-9d85-b443176966a1%40sigaev.ru
Author: Teodor Sigaev
Reviewed-by: Aleksander Alekseev, Tom Lane, Michael Paquier, Roman Zharkov
Reviewed-by: Andrei Lepikhov, Pavel Borisov, Jian He, Alexander Lakhin
Reviewed-by: Artur Zakirov
---
src/backend/access/transam/xact.c | 10 +
src/backend/utils/cache/typcache.c | 359 +++++++++++++++---
src/include/utils/typcache.h | 4 +
src/test/modules/Makefile | 4 +-
src/test/modules/meson.build | 1 +
src/test/modules/typcache/.gitignore | 4 +
src/test/modules/typcache/Makefile | 28 ++
.../expected/typcache_rel_type_cache.out | 34 ++
src/test/modules/typcache/meson.build | 16 +
.../typcache/sql/typcache_rel_type_cache.sql | 18 +
src/tools/pgindent/typedefs.list | 1 +
11 files changed, 433 insertions(+), 46 deletions(-)
create mode 100644 src/test/modules/typcache/.gitignore
create mode 100644 src/test/modules/typcache/Makefile
create mode 100644 src/test/modules/typcache/expected/typcache_rel_type_cache.out
create mode 100644 src/test/modules/typcache/meson.build
create mode 100644 src/test/modules/typcache/sql/typcache_rel_type_cache.sql
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 87700c7c5c7..b0b05e28790 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -70,6 +70,7 @@
#include "utils/snapmgr.h"
#include "utils/timeout.h"
#include "utils/timestamp.h"
+#include "utils/typcache.h"
/*
* User-tweakable parameters
@@ -2407,6 +2408,9 @@ CommitTransaction(void)
/* Clean up the relation cache */
AtEOXact_RelationCache(true);
+ /* Clean up the type cache */
+ AtEOXact_TypeCache();
+
/*
* Make catalog changes visible to all backends. This has to happen after
* relcache references are dropped (see comments for
@@ -2709,6 +2713,9 @@ PrepareTransaction(void)
/* Clean up the relation cache */
AtEOXact_RelationCache(true);
+ /* Clean up the type cache */
+ AtEOXact_TypeCache();
+
/* notify doesn't need a postprepare call */
PostPrepare_PgStat();
@@ -2951,6 +2958,7 @@ AbortTransaction(void)
false, true);
AtEOXact_Buffers(false);
AtEOXact_RelationCache(false);
+ AtEOXact_TypeCache();
AtEOXact_Inval(false);
AtEOXact_MultiXact();
ResourceOwnerRelease(TopTransactionResourceOwner,
@@ -5153,6 +5161,7 @@ CommitSubTransaction(void)
true, false);
AtEOSubXact_RelationCache(true, s->subTransactionId,
s->parent->subTransactionId);
+ AtEOSubXact_TypeCache();
AtEOSubXact_Inval(true);
AtSubCommit_smgr();
@@ -5328,6 +5337,7 @@ AbortSubTransaction(void)
AtEOSubXact_RelationCache(false, s->subTransactionId,
s->parent->subTransactionId);
+ AtEOSubXact_TypeCache();
AtEOSubXact_Inval(false);
ResourceOwnerRelease(s->curTransactionOwner,
RESOURCE_RELEASE_LOCKS,
diff --git a/src/backend/utils/cache/typcache.c b/src/backend/utils/cache/typcache.c
index f142624ad2e..1972bd1944b 100644
--- a/src/backend/utils/cache/typcache.c
+++ b/src/backend/utils/cache/typcache.c
@@ -66,6 +66,7 @@
#include "utils/builtins.h"
#include "utils/catcache.h"
#include "utils/fmgroids.h"
+#include "utils/injection_point.h"
#include "utils/inval.h"
#include "utils/lsyscache.h"
#include "utils/memutils.h"
@@ -77,6 +78,20 @@
/* The main type cache hashtable searched by lookup_type_cache */
static HTAB *TypeCacheHash = NULL;
+/*
+ * The mapping of relation's OID to the corresponding composite type OID.
+ * We're keeping the map entry when the corresponding typentry has something
+ * to clear i.e it has either TCFLAGS_HAVE_PG_TYPE_DATA, or
+ * TCFLAGS_OPERATOR_FLAGS, or tupdesc.
+ */
+static HTAB *RelIdToTypeIdCacheHash = NULL;
+
+typedef struct RelIdToTypeIdCacheEntry
+{
+ Oid relid; /* OID of the relation */
+ Oid composite_typid; /* OID of the relation's composite type */
+} RelIdToTypeIdCacheEntry;
+
/* List of type cache entries for domain types */
static TypeCacheEntry *firstDomainTypeEntry = NULL;
@@ -208,6 +223,10 @@ typedef struct SharedTypmodTableEntry
dsa_pointer shared_tupdesc;
} SharedTypmodTableEntry;
+static Oid *in_progress_list;
+static int in_progress_list_len;
+static int in_progress_list_maxlen;
+
/*
* A comparator function for SharedRecordTableKey.
*/
@@ -329,6 +348,8 @@ static void shared_record_typmod_registry_detach(dsm_segment *segment,
static TupleDesc find_or_make_matching_shared_tupledesc(TupleDesc tupdesc);
static dsa_pointer share_tupledesc(dsa_area *area, TupleDesc tupdesc,
uint32 typmod);
+static void insert_rel_type_cache_if_needed(TypeCacheEntry *typentry);
+static void delete_rel_type_cache_if_needed(TypeCacheEntry *typentry);
/*
@@ -366,11 +387,13 @@ lookup_type_cache(Oid type_id, int flags)
{
TypeCacheEntry *typentry;
bool found;
+ int in_progress_offset;
if (TypeCacheHash == NULL)
{
/* First time through: initialize the hash table */
HASHCTL ctl;
+ int allocsize;
ctl.keysize = sizeof(Oid);
ctl.entrysize = sizeof(TypeCacheEntry);
@@ -385,6 +408,13 @@ lookup_type_cache(Oid type_id, int flags)
TypeCacheHash = hash_create("Type information cache", 64,
&ctl, HASH_ELEM | HASH_FUNCTION);
+ Assert(RelIdToTypeIdCacheHash == NULL);
+
+ ctl.keysize = sizeof(Oid);
+ ctl.entrysize = sizeof(RelIdToTypeIdCacheEntry);
+ RelIdToTypeIdCacheHash = hash_create("Map from relid to OID of cached composite type", 64,
+ &ctl, HASH_ELEM | HASH_BLOBS);
+
/* Also set up callbacks for SI invalidations */
CacheRegisterRelcacheCallback(TypeCacheRelCallback, (Datum) 0);
CacheRegisterSyscacheCallback(TYPEOID, TypeCacheTypCallback, (Datum) 0);
@@ -394,8 +424,32 @@ lookup_type_cache(Oid type_id, int flags)
/* Also make sure CacheMemoryContext exists */
if (!CacheMemoryContext)
CreateCacheMemoryContext();
+
+ /*
+ * reserve enough in_progress_list slots for many cases
+ */
+ allocsize = 4;
+ in_progress_list =
+ MemoryContextAlloc(CacheMemoryContext,
+ allocsize * sizeof(*in_progress_list));
+ in_progress_list_maxlen = allocsize;
}
+ Assert(TypeCacheHash != NULL && RelIdToTypeIdCacheHash != NULL);
+
+ /* Register to catch invalidation messages */
+ if (in_progress_list_len >= in_progress_list_maxlen)
+ {
+ int allocsize;
+
+ allocsize = in_progress_list_maxlen * 2;
+ in_progress_list = repalloc(in_progress_list,
+ allocsize * sizeof(*in_progress_list));
+ in_progress_list_maxlen = allocsize;
+ }
+ in_progress_offset = in_progress_list_len++;
+ in_progress_list[in_progress_offset] = type_id;
+
/* Try to look up an existing entry */
typentry = (TypeCacheEntry *) hash_search(TypeCacheHash,
&type_id,
@@ -896,6 +950,13 @@ lookup_type_cache(Oid type_id, int flags)
load_domaintype_info(typentry);
}
+ INJECTION_POINT("typecache-before-rel-type-cache-insert");
+
+ Assert(in_progress_offset + 1 == in_progress_list_len);
+ in_progress_list_len--;
+
+ insert_rel_type_cache_if_needed(typentry);
+
return typentry;
}
@@ -2290,6 +2351,53 @@ SharedRecordTypmodRegistryAttach(SharedRecordTypmodRegistry *registry)
CurrentSession->shared_typmod_table = typmod_table;
}
+/*
+ * InvalidateCompositeTypeCacheEntry
+ * Invalidate particular TypeCacheEntry on Relcache inval callback
+ *
+ * Delete the cached tuple descriptor (if any) for the given composite
+ * type, and reset whatever info we have cached about the composite type's
+ * comparability.
+ */
+static void
+InvalidateCompositeTypeCacheEntry(TypeCacheEntry *typentry)
+{
+ bool hadTupDescOrOpclass;
+
+ Assert(typentry->typtype == TYPTYPE_COMPOSITE &&
+ OidIsValid(typentry->typrelid));
+
+ hadTupDescOrOpclass = (typentry->tupDesc != NULL) ||
+ (typentry->flags & TCFLAGS_OPERATOR_FLAGS);
+
+ /* Delete tupdesc if we have it */
+ if (typentry->tupDesc != NULL)
+ {
+ /*
+ * Release our refcount and free the tupdesc if none remain. We can't
+ * use DecrTupleDescRefCount here because this reference is not logged
+ * by the current resource owner.
+ */
+ Assert(typentry->tupDesc->tdrefcount > 0);
+ if (--typentry->tupDesc->tdrefcount == 0)
+ FreeTupleDesc(typentry->tupDesc);
+ typentry->tupDesc = NULL;
+
+ /*
+ * Also clear tupDesc_identifier, so that anyone watching it will
+ * realize that the tupdesc has changed.
+ */
+ typentry->tupDesc_identifier = 0;
+ }
+
+ /* Reset equality/comparison/hashing validity information */
+ typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
+
+ /* Call delete_rel_type_cache() if we actually cleared something */
+ if (hadTupDescOrOpclass)
+ delete_rel_type_cache_if_needed(typentry);
+}
+
/*
* TypeCacheRelCallback
* Relcache inval callback function
@@ -2299,63 +2407,55 @@ SharedRecordTypmodRegistryAttach(SharedRecordTypmodRegistry *registry)
* whatever info we have cached about the composite type's comparability.
*
* This is called when a relcache invalidation event occurs for the given
- * relid. We must scan the whole typcache hash since we don't know the
- * type OID corresponding to the relid. We could do a direct search if this
- * were a syscache-flush callback on pg_type, but then we would need all
- * ALTER-TABLE-like commands that could modify a rowtype to issue syscache
- * invals against the rel's pg_type OID. The extra SI signaling could very
- * well cost more than we'd save, since in most usages there are not very
- * many entries in a backend's typcache. The risk of bugs-of-omission seems
- * high, too.
- *
- * Another possibility, with only localized impact, is to maintain a second
- * hashtable that indexes composite-type typcache entries by their typrelid.
- * But it's still not clear it's worth the trouble.
+ * relid. We can't use syscache to find a type corresponding to the given
+ * relation because the code can be called outside of transaction. Thus, we
+ * use the RelIdToTypeIdCacheHash map to locate appropriate typcache entry.
*/
static void
TypeCacheRelCallback(Datum arg, Oid relid)
{
- HASH_SEQ_STATUS status;
TypeCacheEntry *typentry;
- /* TypeCacheHash must exist, else this callback wouldn't be registered */
- hash_seq_init(&status, TypeCacheHash);
- while ((typentry = (TypeCacheEntry *) hash_seq_search(&status)) != NULL)
+ /*
+ * RelIdToTypeIdCacheHash and TypeCacheHash should exist, otherwise this
+ * callback wouldn't be registered
+ */
+ if (OidIsValid(relid))
{
- if (typentry->typtype == TYPTYPE_COMPOSITE)
+ RelIdToTypeIdCacheEntry *relentry;
+
+ /*
+ * Find an RelIdToTypeIdCacheHash entry, which should exist as soon as
+ * corresponding typcache entry has something to clean.
+ */
+ relentry = (RelIdToTypeIdCacheEntry *) hash_search(RelIdToTypeIdCacheHash,
+ &relid,
+ HASH_FIND, NULL);
+
+ if (relentry != NULL)
{
- /* Skip if no match, unless we're zapping all composite types */
- if (relid != typentry->typrelid && relid != InvalidOid)
- continue;
+ typentry = (TypeCacheEntry *) hash_search(TypeCacheHash,
+ &relentry->composite_typid,
+ HASH_FIND, NULL);
- /* Delete tupdesc if we have it */
- if (typentry->tupDesc != NULL)
+ if (typentry != NULL)
{
- /*
- * Release our refcount, and free the tupdesc if none remain.
- * (Can't use DecrTupleDescRefCount because this reference is
- * not logged in current resource owner.)
- */
- Assert(typentry->tupDesc->tdrefcount > 0);
- if (--typentry->tupDesc->tdrefcount == 0)
- FreeTupleDesc(typentry->tupDesc);
- typentry->tupDesc = NULL;
+ Assert(typentry->typtype == TYPTYPE_COMPOSITE);
+ Assert(relid == typentry->typrelid);
- /*
- * Also clear tupDesc_identifier, so that anything watching
- * that will realize that the tupdesc has possibly changed.
- * (Alternatively, we could specify that to detect possible
- * tupdesc change, one must check for tupDesc != NULL as well
- * as tupDesc_identifier being the same as what was previously
- * seen. That seems error-prone.)
- */
- typentry->tupDesc_identifier = 0;
+ InvalidateCompositeTypeCacheEntry(typentry);
}
-
- /* Reset equality/comparison/hashing validity information */
- typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
}
- else if (typentry->typtype == TYPTYPE_DOMAIN)
+
+ /*
+ * Visit all the domain types sequentially. Typically, this shouldn't
+ * affect performance since domain types are less tended to bloat.
+ * Domain types are created manually, unlike composite types which are
+ * automatically created for every temporary table.
+ */
+ for (typentry = firstDomainTypeEntry;
+ typentry != NULL;
+ typentry = typentry->nextDomain)
{
/*
* If it's domain over composite, reset flags. (We don't bother
@@ -2367,6 +2467,36 @@ TypeCacheRelCallback(Datum arg, Oid relid)
typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
}
}
+ else
+ {
+ HASH_SEQ_STATUS status;
+
+ /*
+ * Relid is invalid. By convention, we need to reset all composite
+ * types in cache. Also, we should reset flags for domain types, and
+ * we loop over all entries in hash, so, do it in a single scan.
+ */
+ hash_seq_init(&status, TypeCacheHash);
+ while ((typentry = (TypeCacheEntry *) hash_seq_search(&status)) != NULL)
+ {
+ if (typentry->typtype == TYPTYPE_COMPOSITE)
+ {
+ InvalidateCompositeTypeCacheEntry(typentry);
+ }
+ else if (typentry->typtype == TYPTYPE_DOMAIN)
+ {
+ /*
+ * If it's domain over composite, reset flags. (We don't
+ * bother trying to determine whether the specific base type
+ * needs a reset.) Note that if we haven't determined whether
+ * the base type is composite, we don't need to reset
+ * anything.
+ */
+ if (typentry->flags & TCFLAGS_DOMAIN_BASE_IS_COMPOSITE)
+ typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
+ }
+ }
+ }
}
/*
@@ -2397,6 +2527,8 @@ TypeCacheTypCallback(Datum arg, int cacheid, uint32 hashvalue)
while ((typentry = (TypeCacheEntry *) hash_seq_search(&status)) != NULL)
{
+ bool hadPgTypeData = (typentry->flags & TCFLAGS_HAVE_PG_TYPE_DATA);
+
Assert(hashvalue == 0 || typentry->type_id_hash == hashvalue);
/*
@@ -2406,6 +2538,13 @@ TypeCacheTypCallback(Datum arg, int cacheid, uint32 hashvalue)
*/
typentry->flags &= ~(TCFLAGS_HAVE_PG_TYPE_DATA |
TCFLAGS_CHECKED_DOMAIN_CONSTRAINTS);
+
+ /*
+ * Call delete_rel_type_cache() if we cleaned
+ * TCFLAGS_HAVE_PG_TYPE_DATA flag previously.
+ */
+ if (hadPgTypeData)
+ delete_rel_type_cache_if_needed(typentry);
}
}
@@ -2914,3 +3053,135 @@ shared_record_typmod_registry_detach(dsm_segment *segment, Datum datum)
}
CurrentSession->shared_typmod_registry = NULL;
}
+
+/*
+ * Insert RelIdToTypeIdCacheHash entry if needed.
+ */
+static void
+insert_rel_type_cache_if_needed(TypeCacheEntry *typentry)
+{
+ /* Immediately quit for non-composite types */
+ if (typentry->typtype != TYPTYPE_COMPOSITE)
+ return;
+
+ /* typrelid should be given for composite types */
+ Assert(OidIsValid(typentry->typrelid));
+
+ /*
+ * Insert a RelIdToTypeIdCacheHash entry if the typentry have any
+ * information indicating it should be here.
+ */
+ if ((typentry->flags & TCFLAGS_HAVE_PG_TYPE_DATA) ||
+ (typentry->flags & TCFLAGS_OPERATOR_FLAGS) ||
+ typentry->tupDesc != NULL)
+ {
+ RelIdToTypeIdCacheEntry *relentry;
+ bool found;
+
+ relentry = (RelIdToTypeIdCacheEntry *) hash_search(RelIdToTypeIdCacheHash,
+ &typentry->typrelid,
+ HASH_ENTER, &found);
+ relentry->relid = typentry->typrelid;
+ relentry->composite_typid = typentry->type_id;
+ }
+}
+
+/*
+ * Delete entry RelIdToTypeIdCacheHash if needed after resetting of the
+ * TCFLAGS_HAVE_PG_TYPE_DATA flag, or any of TCFLAGS_OPERATOR_FLAGS,
+ * or tupDesc.
+ */
+static void
+delete_rel_type_cache_if_needed(TypeCacheEntry *typentry)
+{
+#ifdef USE_ASSERT_CHECKING
+ int i;
+ bool is_in_progress = false;
+
+ for (i = 0; i < in_progress_list_len; i++)
+ {
+ if (in_progress_list[i] == typentry->type_id)
+ {
+ is_in_progress = true;
+ break;
+ }
+ }
+#endif
+
+ /* Immediately quit for non-composite types */
+ if (typentry->typtype != TYPTYPE_COMPOSITE)
+ return;
+
+ /* typrelid should be given for composite types */
+ Assert(OidIsValid(typentry->typrelid));
+
+ /*
+ * Delete a RelIdToTypeIdCacheHash entry if the typentry doesn't have any
+ * information indicating entry should be still there.
+ */
+ if (!(typentry->flags & TCFLAGS_HAVE_PG_TYPE_DATA) &&
+ !(typentry->flags & TCFLAGS_OPERATOR_FLAGS) &&
+ typentry->tupDesc == NULL)
+ {
+ bool found;
+
+ (void) hash_search(RelIdToTypeIdCacheHash,
+ &typentry->typrelid,
+ HASH_REMOVE, &found);
+ Assert(found || is_in_progress);
+ }
+ else
+ {
+#ifdef USE_ASSERT_CHECKING
+ /*
+ * In assert-enabled builds otherwise check for RelIdToTypeIdCacheHash
+ * entry if it should exist.
+ */
+ bool found;
+
+ if (!is_in_progress)
+ {
+ (void) hash_search(RelIdToTypeIdCacheHash,
+ &typentry->typrelid,
+ HASH_FIND, &found);
+ Assert(found);
+ }
+#endif
+ }
+}
+
+/*
+ * Add possibly missing RelIdToTypeId entries related to TypeCacheHash
+ * entries, marked as in-progress by lookup_type_cache(). It may happen
+ * in case of an error or interruption during the lookup_type_cache() call.
+ */
+static void
+finalize_in_progress_typentries(void)
+{
+ int i;
+
+ for (i = 0; i < in_progress_list_len; i++)
+ {
+ TypeCacheEntry *typentry;
+
+ typentry = (TypeCacheEntry *) hash_search(TypeCacheHash,
+ &in_progress_list[i],
+ HASH_FIND, NULL);
+ if (typentry)
+ insert_rel_type_cache_if_needed(typentry);
+ }
+
+ in_progress_list_len = 0;
+}
+
+void
+AtEOXact_TypeCache(void)
+{
+ finalize_in_progress_typentries();
+}
+
+void
+AtEOSubXact_TypeCache(void)
+{
+ finalize_in_progress_typentries();
+}
diff --git a/src/include/utils/typcache.h b/src/include/utils/typcache.h
index f506cc4aa35..f3d73ecee3a 100644
--- a/src/include/utils/typcache.h
+++ b/src/include/utils/typcache.h
@@ -207,4 +207,8 @@ extern void SharedRecordTypmodRegistryInit(SharedRecordTypmodRegistry *,
extern void SharedRecordTypmodRegistryAttach(SharedRecordTypmodRegistry *);
+extern void AtEOXact_TypeCache(void);
+
+extern void AtEOSubXact_TypeCache(void);
+
#endif /* TYPCACHE_H */
diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile
index 256799f520a..c0d3cf0e14b 100644
--- a/src/test/modules/Makefile
+++ b/src/test/modules/Makefile
@@ -43,9 +43,9 @@ SUBDIRS = \
ifeq ($(enable_injection_points),yes)
-SUBDIRS += injection_points gin
+SUBDIRS += injection_points gin typcache
else
-ALWAYS_SUBDIRS += injection_points gin
+ALWAYS_SUBDIRS += injection_points gin typcache
endif
ifeq ($(with_ssl),openssl)
diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build
index d8fe059d236..c829b619530 100644
--- a/src/test/modules/meson.build
+++ b/src/test/modules/meson.build
@@ -36,6 +36,7 @@ subdir('test_rls_hooks')
subdir('test_shm_mq')
subdir('test_slru')
subdir('test_tidstore')
+subdir('typcache')
subdir('unsafe_tests')
subdir('worker_spi')
subdir('xid_wraparound')
diff --git a/src/test/modules/typcache/.gitignore b/src/test/modules/typcache/.gitignore
new file mode 100644
index 00000000000..5dcb3ff9723
--- /dev/null
+++ b/src/test/modules/typcache/.gitignore
@@ -0,0 +1,4 @@
+# Generated subdirectories
+/log/
+/results/
+/tmp_check/
diff --git a/src/test/modules/typcache/Makefile b/src/test/modules/typcache/Makefile
new file mode 100644
index 00000000000..1f03de83890
--- /dev/null
+++ b/src/test/modules/typcache/Makefile
@@ -0,0 +1,28 @@
+# src/test/modules/typcache/Makefile
+
+EXTRA_INSTALL = src/test/modules/injection_points
+
+REGRESS = typcache_rel_type_cache
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = src/test/modules/typcache
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+
+# XXX: This test is conditional on enable_injection_points in the
+# parent Makefile, so we should never get here in the first place if
+# injection points are not enabled. But the buildfarm 'misc-check'
+# step doesn't pay attention to the if-condition in the parent
+# Makefile. To work around that, disable running the test here too.
+ifeq ($(enable_injection_points),yes)
+include $(top_srcdir)/contrib/contrib-global.mk
+else
+check:
+ @echo "injection points are disabled in this build"
+endif
+
+endif
diff --git a/src/test/modules/typcache/expected/typcache_rel_type_cache.out b/src/test/modules/typcache/expected/typcache_rel_type_cache.out
new file mode 100644
index 00000000000..b113e0bbd5d
--- /dev/null
+++ b/src/test/modules/typcache/expected/typcache_rel_type_cache.out
@@ -0,0 +1,34 @@
+--
+-- This test checks that lookup_type_cache() can correctly handle an
+-- interruption. We use the injection point to simulate an error but note
+-- that a similar situation could happen due to user query interruption.
+-- Despite the interruption, a map entry from the relation oid to type cache
+-- entry should be created. This is validated by subsequent modification of
+-- the table schema, then type casts which use new schema implying
+-- successful type cache invalidation by relation oid.
+--
+CREATE EXTENSION injection_points;
+CREATE TABLE t (i int);
+SELECT injection_points_attach('typecache-before-rel-type-cache-insert', 'error');
+ injection_points_attach
+-------------------------
+
+(1 row)
+
+SELECT '(1)'::t;
+ERROR: error triggered for injection point typecache-before-rel-type-cache-insert
+LINE 1: SELECT '(1)'::t;
+ ^
+SELECT injection_points_detach('typecache-before-rel-type-cache-insert');
+ injection_points_detach
+-------------------------
+
+(1 row)
+
+ALTER TABLE t ADD COLUMN j int;
+SELECT '(1,2)'::t;
+ t
+-------
+ (1,2)
+(1 row)
+
diff --git a/src/test/modules/typcache/meson.build b/src/test/modules/typcache/meson.build
new file mode 100644
index 00000000000..cb2e34c0d2b
--- /dev/null
+++ b/src/test/modules/typcache/meson.build
@@ -0,0 +1,16 @@
+# Copyright (c) 2022-2024, PostgreSQL Global Development Group
+
+if not get_option('injection_points')
+ subdir_done()
+endif
+
+tests += {
+ 'name': 'typcache',
+ 'sd': meson.current_source_dir(),
+ 'bd': meson.current_build_dir(),
+ 'regress': {
+ 'sql': [
+ 'typcache_rel_type_cache',
+ ],
+ },
+}
diff --git a/src/test/modules/typcache/sql/typcache_rel_type_cache.sql b/src/test/modules/typcache/sql/typcache_rel_type_cache.sql
new file mode 100644
index 00000000000..2c0a434d988
--- /dev/null
+++ b/src/test/modules/typcache/sql/typcache_rel_type_cache.sql
@@ -0,0 +1,18 @@
+--
+-- This test checks that lookup_type_cache() can correctly handle an
+-- interruption. We use the injection point to simulate an error but note
+-- that a similar situation could happen due to user query interruption.
+-- Despite the interruption, a map entry from the relation oid to type cache
+-- entry should be created. This is validated by subsequent modification of
+-- the table schema, then type casts which use new schema implying
+-- successful type cache invalidation by relation oid.
+--
+
+CREATE EXTENSION injection_points;
+
+CREATE TABLE t (i int);
+SELECT injection_points_attach('typecache-before-rel-type-cache-insert', 'error');
+SELECT '(1)'::t;
+SELECT injection_points_detach('typecache-before-rel-type-cache-insert');
+ALTER TABLE t ADD COLUMN j int;
+SELECT '(1,2)'::t;
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 57de1acff3a..9b3e7fd104b 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2378,6 +2378,7 @@ RelFileLocator
RelFileLocatorBackend
RelFileNumber
RelIdCacheEnt
+RelIdToTypeIdCacheEntry
RelInfo
RelInfoArr
RelMapFile
--
2.39.5 (Apple Git-154)
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2024-10-25 08:35 Andres Freund <[email protected]>
parent: Alexander Korotkov <[email protected]>
1 sibling, 1 reply; 55+ messages in thread
From: Andres Freund @ 2024-10-25 08:35 UTC (permalink / raw)
To: Alexander Korotkov <[email protected]>; +Cc: Pavel Borisov <[email protected]>; Dagfinn Ilmari Mannsåker <[email protected]>; Andrei Lepikhov <[email protected]>; jian he <[email protected]>; Artur Zakirov <[email protected]>; Alexander Lakhin <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
Hi,
On 2024-10-22 20:33:24 +0300, Alexander Korotkov wrote:
> Thank you, Pavel! 0001 revised according to your suggestion.
Starting with this commit CI fails.
https://cirrus-ci.com/task/6668851469877248
https://api.cirrus-ci.com/v1/artifact/task/6668851469877248/testrun/build/testrun/regress-running/re...
diff -U3 /tmp/cirrus-ci-build/src/test/regress/expected/inherit.out /tmp/cirrus-ci-build/build/testrun/regress-running/regress/results/inherit.out
--- /tmp/cirrus-ci-build/src/test/regress/expected/inherit.out 2024-10-24 11:38:43.829712000 +0000
+++ /tmp/cirrus-ci-build/build/testrun/regress-running/regress/results/inherit.out 2024-10-24 11:44:57.154238000 +0000
@@ -1338,14 +1338,9 @@
ERROR: cannot drop inherited constraint "f1_pos" of relation "p1_c1"
alter table p1 drop constraint f1_pos;
\d p1_c1
- Table "public.p1_c1"
- Column | Type | Collation | Nullable | Default
---------+---------+-----------+----------+---------
- f1 | integer | | |
-Check constraints:
- "f1_pos" CHECK (f1 > 0)
-Inherits: p1
-
+ERROR: error triggered for injection point typecache-before-rel-type-cache-insert
+LINE 4: ORDER BY 1;
+ ^
drop table p1 cascade;
NOTICE: drop cascades to table p1_c1
create table p1(f1 int constraint f1_pos CHECK (f1 > 0));
Greetings,
Andres
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2024-10-25 09:48 Alexander Korotkov <[email protected]>
parent: Andres Freund <[email protected]>
0 siblings, 1 reply; 55+ messages in thread
From: Alexander Korotkov @ 2024-10-25 09:48 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Pavel Borisov <[email protected]>; Dagfinn Ilmari Mannsåker <[email protected]>; Andrei Lepikhov <[email protected]>; jian he <[email protected]>; Artur Zakirov <[email protected]>; Alexander Lakhin <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
On Fri, Oct 25, 2024 at 11:35 AM Andres Freund <[email protected]> wrote:
> On 2024-10-22 20:33:24 +0300, Alexander Korotkov wrote:
> > Thank you, Pavel! 0001 revised according to your suggestion.
>
> Starting with this commit CI fails.
>
> https://cirrus-ci.com/task/6668851469877248
> https://api.cirrus-ci.com/v1/artifact/task/6668851469877248/testrun/build/testrun/regress-running/re...
>
> diff -U3 /tmp/cirrus-ci-build/src/test/regress/expected/inherit.out /tmp/cirrus-ci-build/build/testrun/regress-running/regress/results/inherit.out
> --- /tmp/cirrus-ci-build/src/test/regress/expected/inherit.out 2024-10-24 11:38:43.829712000 +0000
> +++ /tmp/cirrus-ci-build/build/testrun/regress-running/regress/results/inherit.out 2024-10-24 11:44:57.154238000 +0000
> @@ -1338,14 +1338,9 @@
> ERROR: cannot drop inherited constraint "f1_pos" of relation "p1_c1"
> alter table p1 drop constraint f1_pos;
> \d p1_c1
> - Table "public.p1_c1"
> - Column | Type | Collation | Nullable | Default
> ---------+---------+-----------+----------+---------
> - f1 | integer | | |
> -Check constraints:
> - "f1_pos" CHECK (f1 > 0)
> -Inherits: p1
> -
> +ERROR: error triggered for injection point typecache-before-rel-type-cache-insert
> +LINE 4: ORDER BY 1;
> + ^
> drop table p1 cascade;
> NOTICE: drop cascades to table p1_c1
> create table p1(f1 int constraint f1_pos CHECK (f1 > 0));
Thank you for reporting this.
Looks weird that injection point, which isn't used in these tests, got
triggered here.
I'm looking into this.
------
Regards,
Alexander Korotkov
Supabase
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2024-10-25 10:13 Alexander Korotkov <[email protected]>
parent: Alexander Korotkov <[email protected]>
0 siblings, 0 replies; 55+ messages in thread
From: Alexander Korotkov @ 2024-10-25 10:13 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Pavel Borisov <[email protected]>; Dagfinn Ilmari Mannsåker <[email protected]>; Andrei Lepikhov <[email protected]>; jian he <[email protected]>; Artur Zakirov <[email protected]>; Alexander Lakhin <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
On Fri, Oct 25, 2024 at 12:48 PM Alexander Korotkov
<[email protected]> wrote:
> On Fri, Oct 25, 2024 at 11:35 AM Andres Freund <[email protected]> wrote:
> > On 2024-10-22 20:33:24 +0300, Alexander Korotkov wrote:
> > > Thank you, Pavel! 0001 revised according to your suggestion.
> >
> > Starting with this commit CI fails.
> >
> > https://cirrus-ci.com/task/6668851469877248
> > https://api.cirrus-ci.com/v1/artifact/task/6668851469877248/testrun/build/testrun/regress-running/re...
> >
> > diff -U3 /tmp/cirrus-ci-build/src/test/regress/expected/inherit.out /tmp/cirrus-ci-build/build/testrun/regress-running/regress/results/inherit.out
> > --- /tmp/cirrus-ci-build/src/test/regress/expected/inherit.out 2024-10-24 11:38:43.829712000 +0000
> > +++ /tmp/cirrus-ci-build/build/testrun/regress-running/regress/results/inherit.out 2024-10-24 11:44:57.154238000 +0000
> > @@ -1338,14 +1338,9 @@
> > ERROR: cannot drop inherited constraint "f1_pos" of relation "p1_c1"
> > alter table p1 drop constraint f1_pos;
> > \d p1_c1
> > - Table "public.p1_c1"
> > - Column | Type | Collation | Nullable | Default
> > ---------+---------+-----------+----------+---------
> > - f1 | integer | | |
> > -Check constraints:
> > - "f1_pos" CHECK (f1 > 0)
> > -Inherits: p1
> > -
> > +ERROR: error triggered for injection point typecache-before-rel-type-cache-insert
> > +LINE 4: ORDER BY 1;
> > + ^
> > drop table p1 cascade;
> > NOTICE: drop cascades to table p1_c1
> > create table p1(f1 int constraint f1_pos CHECK (f1 > 0));
>
> Thank you for reporting this.
> Looks weird that injection point, which isn't used in these tests, got
> triggered here.
> I'm looking into this.
Oh, I forgot to make injection points in typcache_rel_type_cache.sql
local. Thus, it affects concurrent tests. Must be fixed in
aa1e898dea.
------
Regards,
Alexander Korotkov
Supabase
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-07 00:05 Jelte Fennema-Nio <[email protected]>
0 siblings, 2 replies; 55+ messages in thread
From: Jelte Fennema-Nio @ 2025-03-07 00:05 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Anthonin Bonnefoy <[email protected]>; pgsql-hackers
On Tue, 25 Feb 2025 at 02:11, Michael Paquier <[email protected]> wrote:
> Initial digestion has gone well.
One thing I've noticed is that \startpipeline throws warnings when
copy pasting multiple lines. It seems to still execute everything as
expected though. As an example you can copy paste this tiny script:
\startpipeline
select pg_sleep(5) \bind \g
\endpipeline
And then it will show these "extra argument ... ignored" warnings
\startpipeline: extra argument "select" ignored
\startpipeline: extra argument "pg_sleep(5)" ignored
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-07 08:08 Anthonin Bonnefoy <[email protected]>
parent: Jelte Fennema-Nio <[email protected]>
1 sibling, 0 replies; 55+ messages in thread
From: Anthonin Bonnefoy @ 2025-03-07 08:08 UTC (permalink / raw)
To: Jelte Fennema-Nio <[email protected]>; +Cc: Michael Paquier <[email protected]>; pgsql-hackers
On Thu, Mar 6, 2025 at 5:20 AM Michael Paquier <[email protected]> wrote:
> That was not a test case we had in mind originally here, but if it is
> possible to keep the implementation simple while supporting your
> demand, well, let's do it. If it's not that straight-forward, let's
> use the new meta-command, forbidding \g and \gx based on your
> arguments from upthread.
I think the new meta-command is a separate issue from allowing ';' to
push in a pipeline. Any time there's a change or an additional format
option added to \g, it will need to be forbidden for pipelining. The
\sendpipeline meta-command will help keep those exceptions low since
the whole \g will be forbidden.
Another possible option would be to allow both \g and \gx, but send a
warning like "printing options within a pipeline will be ignored" if
those options are used, similar to "SET LOCAL" warning when done
outside of a transaction block. That would have the benefit of making
existing scripts using \g and \gx compatible.
For using ';' to push commands in a pipeline, I think it should be
fairly straightforward. I can try to work on that next week (I'm
currently chasing a weird memory context bug that I need to finish
first).
On Fri, Mar 7, 2025 at 1:05 AM Jelte Fennema-Nio <[email protected]> wrote:
> One thing I've noticed is that \startpipeline throws warnings when
> copy pasting multiple lines. It seems to still execute everything as
> expected though. As an example you can copy paste this tiny script:
>
> \startpipeline
> select pg_sleep(5) \bind \g
> \endpipeline
>
> And then it will show these "extra argument ... ignored" warnings
>
> \startpipeline: extra argument "select" ignored
> \startpipeline: extra argument "pg_sleep(5)" ignored
It looks like an issue with libreadline. At least, I've been able to
reproduce the warnings and 'readline(prompt);' returns everything as a
single line, with the \n inside the string. This explains why what is
after \startpipeline is processed as arguments. This can also be done
with:
select 1 \bind \g
select 2 \bind \g
And somehow, I couldn't reproduce the issue anymore once I've compiled
and installed libreadline with debug symbols.
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-07 22:31 Daniel Verite <[email protected]>
parent: Jelte Fennema-Nio <[email protected]>
1 sibling, 1 reply; 55+ messages in thread
From: Daniel Verite @ 2025-03-07 22:31 UTC (permalink / raw)
To: Jelte Fennema-Nio <[email protected]>; +Cc: Michael Paquier <[email protected]>; Anthonin Bonnefoy <[email protected]>; pgsql-hackers
Jelte Fennema-Nio wrote:
> As an example you can copy paste this tiny script:
>
> \startpipeline
> select pg_sleep(5) \bind \g
> \endpipeline
>
> And then it will show these "extra argument ... ignored" warnings
>
> \startpipeline: extra argument "select" ignored
> \startpipeline: extra argument "pg_sleep(5)" ignored
It happens with other metacommands as well, and appears
to depend on a readline option that is "on" by default since
readline-8.1 [1]
enable-bracketed-paste
When set to ‘On’, Readline configures the terminal to insert each
paste into the editing buffer as a single string of characters,
instead of treating each character as if it had been read from the
keyboard. This is called putting the terminal into bracketed paste
mode; it prevents Readline from executing any editing commands
bound to key sequences appearing in the pasted text. The default
is ‘On’.
This behavior of the metacommand complaining about arguments
on the next line also happens if using \e and typing this sequence
of commands in the editor. In that case readline is not involved.
There might be something to improve here, because
a metacommand cannot take its argument from the next line,
and yet that's what the error messages somewhat imply.
But that issue is not related to the new pipeline metacommands.
[1]
https://tiswww.case.edu/php/chet/readline/readline.html#index-enable_002dbracketed_002dpaste
Best regards,
--
Daniel Vérité
https://postgresql.verite.pro/
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-17 09:50 Anthonin Bonnefoy <[email protected]>
parent: Daniel Verite <[email protected]>
0 siblings, 3 replies; 55+ messages in thread
From: Anthonin Bonnefoy @ 2025-03-17 09:50 UTC (permalink / raw)
To: Daniel Verite <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; Michael Paquier <[email protected]>; pgsql-hackers
Here is a new patch set:
0001: This introduces the \sendpipeline meta-command and forbid \g in
a pipeline. This is to fix the formatting options of \g that are not
supported in a pipeline.
0002: Allows ';' to send a query using extended protocol when within a
pipeline by using PQsendQueryParams with 0 parameters. It is not
possible to send parameters with extended protocol this way and
everything will be propagated through the query string, similar to a
simple query.
Attachments:
[application/octet-stream] v02-0001-psql-Create-new-sendpipeline-meta-command.patch (38.8K, ../../CAO6_Xqq4JEHRaGyC=r+dpi0Ejbp2P3aUo-KfR2NuipCvE-o_ZA@mail.gmail.com/2-v02-0001-psql-Create-new-sendpipeline-meta-command.patch)
download | inline diff:
From 097084856575dbcd53baad01a1b5420b4201036d Mon Sep 17 00:00:00 2001
From: Anthonin Bonnefoy <[email protected]>
Date: Tue, 4 Mar 2025 09:47:45 +0100
Subject: psql: Create new \sendpipeline meta-command
In the initial pipelining support for psql, \g was reused as a way to
push extended query into an ongoing pipeline. However, all related
meta-command options are not applicable in pipeline mode. Pipeline
output only happens during \getresults or \endpipeline and any change to
output options would have been reset.
To avoid possible confusion, a new \sendpipeline meta-command is
introduced, replacing \g when within a pipeline.
---
doc/src/sgml/ref/psql-ref.sgml | 11 +-
src/bin/psql/command.c | 45 +++-
src/bin/psql/help.c | 1 +
src/bin/psql/tab-complete.in.c | 2 +-
src/test/regress/expected/psql.out | 1 +
src/test/regress/expected/psql_pipeline.out | 245 +++++++++++---------
src/test/regress/sql/psql.sql | 1 +
src/test/regress/sql/psql_pipeline.sql | 230 +++++++++---------
8 files changed, 310 insertions(+), 226 deletions(-)
diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml
index cedccc14129..cddf6e07531 100644
--- a/doc/src/sgml/ref/psql-ref.sgml
+++ b/doc/src/sgml/ref/psql-ref.sgml
@@ -3677,6 +3677,7 @@ testdb=> <userinput>\setenv LESS -imx4F</userinput>
<varlistentry id="app-psql-meta-command-pipeline">
<term><literal>\startpipeline</literal></term>
+ <term><literal>\sendpipeline</literal></term>
<term><literal>\syncpipeline</literal></term>
<term><literal>\endpipeline</literal></term>
<term><literal>\flushrequest</literal></term>
@@ -3701,10 +3702,10 @@ testdb=> <userinput>\setenv LESS -imx4F</userinput>
queries need to be sent using the meta-commands
<literal>\bind</literal>, <literal>\bind_named</literal>,
<literal>\close</literal> or <literal>\parse</literal>. While a
- pipeline is ongoing, <literal>\g</literal> will append the current
- query buffer to the pipeline. Other meta-commands like
- <literal>\gx</literal> or <literal>\gdesc</literal> are not allowed
- in pipeline mode.
+ pipeline is ongoing, <literal>\sendpipeline</literal> will append the
+ current query buffer to the pipeline. Other meta-commands like
+ <literal>\g</literal>, <literal>\gx</literal> or <literal>\gdesc</literal>
+ are not allowed in pipeline mode.
</para>
<para>
@@ -3738,7 +3739,7 @@ testdb=> <userinput>\setenv LESS -imx4F</userinput>
Example:
<programlisting>
\startpipeline
-SELECT 1 \bind \g
+SELECT 1 \bind \sendpipeline
\flushrequest
\getresults
\endpipeline
diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c
index fb0b27568c5..7670c20b29e 100644
--- a/src/bin/psql/command.c
+++ b/src/bin/psql/command.c
@@ -137,6 +137,7 @@ static backslashResult exec_command_setenv(PsqlScanState scan_state, bool active
static backslashResult exec_command_sf_sv(PsqlScanState scan_state, bool active_branch,
const char *cmd, bool is_func);
static backslashResult exec_command_startpipeline(PsqlScanState scan_state, bool active_branch);
+static backslashResult exec_command_sendpipeline(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_syncpipeline(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_t(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_T(PsqlScanState scan_state, bool active_branch);
@@ -427,6 +428,8 @@ exec_command(const char *cmd,
status = exec_command_sf_sv(scan_state, active_branch, cmd, false);
else if (strcmp(cmd, "startpipeline") == 0)
status = exec_command_startpipeline(scan_state, active_branch);
+ else if (strcmp(cmd, "sendpipeline") == 0)
+ status = exec_command_sendpipeline(scan_state, active_branch);
else if (strcmp(cmd, "syncpipeline") == 0)
status = exec_command_syncpipeline(scan_state, active_branch);
else if (strcmp(cmd, "t") == 0)
@@ -1734,10 +1737,9 @@ exec_command_g(PsqlScanState scan_state, bool active_branch, const char *cmd)
if (status == PSQL_CMD_SKIP_LINE && active_branch)
{
- if (strcmp(cmd, "gx") == 0 &&
- PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
{
- pg_log_error("\\gx not allowed in pipeline mode");
+ pg_log_error("\\%s not allowed in pipeline mode", cmd);
clean_extended_state();
free(fname);
return PSQL_CMD_ERROR;
@@ -2979,6 +2981,43 @@ exec_command_startpipeline(PsqlScanState scan_state, bool active_branch)
return status;
}
+/*
+ * \sendpipeline -- send an extended query to an ongoing pipeline
+ */
+static backslashResult
+exec_command_sendpipeline(PsqlScanState scan_state, bool active_branch)
+{
+ backslashResult status = PSQL_CMD_SKIP_LINE;
+
+ if (active_branch)
+ {
+ if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ if (pset.send_mode == PSQL_SEND_EXTENDED_QUERY_PREPARED ||
+ pset.send_mode == PSQL_SEND_EXTENDED_QUERY_PARAMS)
+ {
+ status = PSQL_CMD_SEND;
+ }
+ else
+ {
+ pg_log_error("\\sendpipeline must be used after \\bind or \\bind_named");
+ clean_extended_state();
+ return PSQL_CMD_ERROR;
+ }
+ }
+ else
+ {
+ pg_log_error("\\sendpipeline not allowed outside of pipeline mode");
+ clean_extended_state();
+ return PSQL_CMD_ERROR;
+ }
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return status;
+}
+
/*
* \syncpipeline -- send a sync message to an active pipeline
*/
diff --git a/src/bin/psql/help.c b/src/bin/psql/help.c
index 714b8619233..edfc794b76f 100644
--- a/src/bin/psql/help.c
+++ b/src/bin/psql/help.c
@@ -182,6 +182,7 @@ slashUsage(unsigned short int pager)
HELP0(" \\parse STMT_NAME create a prepared statement\n");
HELP0(" \\q quit psql\n");
HELP0(" \\startpipeline enter pipeline mode\n");
+ HELP0(" \\sendpipeline send an extended query to an ongoing pipeline\n");
HELP0(" \\syncpipeline add a synchronisation point to an ongoing pipeline\n");
HELP0(" \\watch [[i=]SEC] [c=N] [m=MIN]\n"
" execute query every SEC seconds, up to N times,\n"
diff --git a/src/bin/psql/tab-complete.in.c b/src/bin/psql/tab-complete.in.c
index 8432be641ac..b2de63a5b74 100644
--- a/src/bin/psql/tab-complete.in.c
+++ b/src/bin/psql/tab-complete.in.c
@@ -1895,7 +1895,7 @@ psql_completion(const char *text, int start, int end)
"\\parse", "\\password", "\\print", "\\prompt", "\\pset",
"\\qecho", "\\quit",
"\\reset",
- "\\s", "\\set", "\\setenv", "\\sf", "\\startpipeline", "\\sv", "\\syncpipeline",
+ "\\s", "\\set", "\\setenv", "\\sf", "\\startpipeline", "\\sendpipeline", "\\sv", "\\syncpipeline",
"\\t", "\\T", "\\timing",
"\\unset",
"\\x",
diff --git a/src/test/regress/expected/psql.out b/src/test/regress/expected/psql.out
index 6543e90de75..ee6f8671bea 100644
--- a/src/test/regress/expected/psql.out
+++ b/src/test/regress/expected/psql.out
@@ -4711,6 +4711,7 @@ invalid command \lo
\sf whole_line
\sv whole_line
\startpipeline
+ \sendpipeline
\syncpipeline
\t arg1
\T arg1
diff --git a/src/test/regress/expected/psql_pipeline.out b/src/test/regress/expected/psql_pipeline.out
index 3df2415a840..53f2edfd986 100644
--- a/src/test/regress/expected/psql_pipeline.out
+++ b/src/test/regress/expected/psql_pipeline.out
@@ -4,7 +4,7 @@
CREATE TABLE psql_pipeline(a INTEGER PRIMARY KEY, s TEXT);
-- Single query
\startpipeline
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
?column?
----------
@@ -13,9 +13,9 @@ SELECT $1 \bind 'val1' \g
-- Multiple queries
\startpipeline
-SELECT $1 \bind 'val1' \g
-SELECT $1, $2 \bind 'val2' 'val3' \g
-SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1 \bind 'val1' \sendpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
\endpipeline
?column?
----------
@@ -35,10 +35,10 @@ SELECT $1, $2 \bind 'val2' 'val3' \g
-- Test \flush
\startpipeline
\flush
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\flush
-SELECT $1, $2 \bind 'val2' 'val3' \g
-SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
\endpipeline
?column?
----------
@@ -63,12 +63,12 @@ SELECT $1, $2 \bind 'val2' 'val3' \g
0
\echo :PIPELINE_RESULT_COUNT
0
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\syncpipeline
\syncpipeline
-SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
\syncpipeline
-SELECT $1, $2 \bind 'val4' 'val5' \g
+SELECT $1, $2 \bind 'val4' 'val5' \sendpipeline
\echo :PIPELINE_COMMAND_COUNT
1
\echo :PIPELINE_SYNC_COUNT
@@ -94,7 +94,7 @@ SELECT $1, $2 \bind 'val4' 'val5' \g
-- \startpipeline should not have any effect if already in a pipeline.
\startpipeline
\startpipeline
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
?column?
----------
@@ -103,24 +103,24 @@ SELECT $1 \bind 'val1' \g
-- Convert an implicit transaction block to an explicit transaction block.
\startpipeline
-INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
-BEGIN \bind \g
-INSERT INTO psql_pipeline VALUES ($1) \bind 2 \g
-ROLLBACK \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \sendpipeline
+BEGIN \bind \sendpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 2 \sendpipeline
+ROLLBACK \bind \sendpipeline
\endpipeline
-- Multiple explicit transactions
\startpipeline
-BEGIN \bind \g
-INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
-ROLLBACK \bind \g
-BEGIN \bind \g
-INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
-COMMIT \bind \g
+BEGIN \bind \sendpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \sendpipeline
+ROLLBACK \bind \sendpipeline
+BEGIN \bind \sendpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \sendpipeline
+COMMIT \bind \sendpipeline
\endpipeline
-- COPY FROM STDIN
\startpipeline
-SELECT $1 \bind 'val1' \g
-COPY psql_pipeline FROM STDIN \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+COPY psql_pipeline FROM STDIN \bind \sendpipeline
\endpipeline
?column?
----------
@@ -129,8 +129,8 @@ COPY psql_pipeline FROM STDIN \bind \g
-- COPY FROM STDIN with \flushrequest + \getresults
\startpipeline
-SELECT $1 \bind 'val1' \g
-COPY psql_pipeline FROM STDIN \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+COPY psql_pipeline FROM STDIN \bind \sendpipeline
\flushrequest
\getresults
?column?
@@ -142,8 +142,8 @@ message type 0x5a arrived from server while idle
\endpipeline
-- COPY FROM STDIN with \syncpipeline + \getresults
\startpipeline
-SELECT $1 \bind 'val1' \g
-COPY psql_pipeline FROM STDIN \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+COPY psql_pipeline FROM STDIN \bind \sendpipeline
\syncpipeline
\getresults
?column?
@@ -154,8 +154,8 @@ COPY psql_pipeline FROM STDIN \bind \g
\endpipeline
-- COPY TO STDOUT
\startpipeline
-SELECT $1 \bind 'val1' \g
-copy psql_pipeline TO STDOUT \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+copy psql_pipeline TO STDOUT \bind \sendpipeline
\endpipeline
?column?
----------
@@ -168,8 +168,8 @@ copy psql_pipeline TO STDOUT \bind \g
4 test4
-- COPY TO STDOUT with \flushrequest + \getresults
\startpipeline
-SELECT $1 \bind 'val1' \g
-copy psql_pipeline TO STDOUT \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+copy psql_pipeline TO STDOUT \bind \sendpipeline
\flushrequest
\getresults
?column?
@@ -184,8 +184,8 @@ copy psql_pipeline TO STDOUT \bind \g
\endpipeline
-- COPY TO STDOUT with \syncpipeline + \getresults
\startpipeline
-SELECT $1 \bind 'val1' \g
-copy psql_pipeline TO STDOUT \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+copy psql_pipeline TO STDOUT \bind \sendpipeline
\syncpipeline
\getresults
?column?
@@ -203,14 +203,14 @@ copy psql_pipeline TO STDOUT \bind \g
SELECT $1 \parse ''
SELECT $1, $2 \parse ''
SELECT $2 \parse pipeline_1
-\bind_named '' 1 2 \g
-\bind_named pipeline_1 2 \g
+\bind_named '' 1 2 \sendpipeline
+\bind_named pipeline_1 2 \sendpipeline
\endpipeline
ERROR: could not determine data type of parameter $1
-- \getresults displays all results preceding a \flushrequest.
\startpipeline
-SELECT $1 \bind 1 \g
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 1 \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\flushrequest
\getresults
?column?
@@ -226,8 +226,8 @@ SELECT $1 \bind 2 \g
\endpipeline
-- \getresults displays all results preceding a \syncpipeline.
\startpipeline
-SELECT $1 \bind 1 \g
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 1 \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\syncpipeline
\getresults
?column?
@@ -245,7 +245,7 @@ SELECT $1 \bind 2 \g
\startpipeline
\getresults
No pending results to get
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\getresults
No pending results to get
\flushrequest
@@ -259,9 +259,9 @@ No pending results to get
No pending results to get
-- \getresults only fetches results preceding a \flushrequest.
\startpipeline
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\flushrequest
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\getresults
?column?
----------
@@ -276,9 +276,9 @@ SELECT $1 \bind 2 \g
-- \getresults only fetches results preceding a \syncpipeline.
\startpipeline
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\syncpipeline
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\getresults
?column?
----------
@@ -294,7 +294,7 @@ SELECT $1 \bind 2 \g
-- Use pipeline with chunked results for both \getresults and \endpipeline.
\startpipeline
\set FETCH_COUNT 10
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\flushrequest
\getresults
?column?
@@ -302,7 +302,7 @@ SELECT $1 \bind 2 \g
2
(1 row)
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
?column?
----------
@@ -312,9 +312,9 @@ SELECT $1 \bind 2 \g
\unset FETCH_COUNT
-- \getresults with specific number of requested results.
\startpipeline
-SELECT $1 \bind 1 \g
-SELECT $1 \bind 2 \g
-SELECT $1 \bind 3 \g
+SELECT $1 \bind 1 \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
+SELECT $1 \bind 3 \sendpipeline
\echo :PIPELINE_SYNC_COUNT
0
\syncpipeline
@@ -330,7 +330,7 @@ SELECT $1 \bind 3 \g
\echo :PIPELINE_RESULT_COUNT
2
-SELECT $1 \bind 4 \g
+SELECT $1 \bind 4 \sendpipeline
\getresults 3
?column?
----------
@@ -354,7 +354,7 @@ SELECT $1 \bind 4 \g
\startpipeline
\syncpipeline
\syncpipeline
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
\flushrequest
\getresults 2
\getresults 1
@@ -366,9 +366,9 @@ SELECT $1 \bind 1 \g
\endpipeline
-- \getresults 0 should get all the results.
\startpipeline
-SELECT $1 \bind 1 \g
-SELECT $1 \bind 2 \g
-SELECT $1 \bind 3 \g
+SELECT $1 \bind 1 \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
+SELECT $1 \bind 3 \sendpipeline
\syncpipeline
\getresults 0
?column?
@@ -398,7 +398,7 @@ cannot send pipeline when not in pipeline mode
\startpipeline
SELECT 1;
PQsendQuery not allowed in pipeline mode
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
?column?
----------
@@ -408,9 +408,9 @@ SELECT $1 \bind 'val1' \g
-- After an aborted pipeline, commands after a \syncpipeline should be
-- displayed.
\startpipeline
-SELECT $1 \bind \g
+SELECT $1 \bind \sendpipeline
\syncpipeline
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
\endpipeline
ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
?column?
@@ -421,23 +421,23 @@ ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
-- For an incorrect number of parameters, the pipeline is aborted and
-- the following queries will not be executed.
\startpipeline
-SELECT \bind 'val1' \g
-SELECT $1 \bind 'val1' \g
+SELECT \bind 'val1' \sendpipeline
+SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
ERROR: bind message supplies 1 parameters, but prepared statement "" requires 0
-- An explicit transaction with an error needs to be rollbacked after
-- the pipeline.
\startpipeline
-BEGIN \bind \g
-INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
-ROLLBACK \bind \g
+BEGIN \bind \sendpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \sendpipeline
+ROLLBACK \bind \sendpipeline
\endpipeline
ERROR: duplicate key value violates unique constraint "psql_pipeline_pkey"
DETAIL: Key (a)=(1) already exists.
ROLLBACK;
-- \watch sends a simple query, something not allowed within a pipeline.
\startpipeline
-SELECT \bind \g
+SELECT \bind \sendpipeline
\watch 1
PQsendQuery not allowed in pipeline mode
@@ -450,7 +450,7 @@ PQsendQuery not allowed in pipeline mode
\startpipeline
SELECT $1 \bind 1 \gdesc
synchronous command execution functions are not allowed in pipeline mode
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
\endpipeline
?column?
----------
@@ -465,19 +465,33 @@ SELECT $1 as k, $2 as l \parse 'second'
\gset not allowed in pipeline mode
\bind_named second 1 2 \gset pref02_ \echo :pref02_i :pref02_j
\gset not allowed in pipeline mode
-\bind_named '' 1 2 \g
+\bind_named '' 1 2 \sendpipeline
\endpipeline
i | j
---+---
1 | 2
(1 row)
--- \gx is not allowed, pipeline should still be usable.
+-- \g and \gx are not allowed, pipeline should still be usable.
\startpipeline
+SELECT $1 \bind 1 \g
+\g not allowed in pipeline mode
+SELECT $1 \bind 1 \g filename
+\g not allowed in pipeline mode
+SELECT $1 \bind 1 \g |cat
+\g not allowed in pipeline mode
+SELECT $1 \bind 1 \g (format=unaligned tuples_only=on)
+\g not allowed in pipeline mode
SELECT $1 \bind 1 \gx
\gx not allowed in pipeline mode
+SELECT $1 \bind 1 \gx filename
+\gx not allowed in pipeline mode
+SELECT $1 \bind 1 \gx |cat
+\gx not allowed in pipeline mode
+SELECT $1 \bind 1 \gx (format=unaligned tuples_only=on)
+\gx not allowed in pipeline mode
\reset
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
\endpipeline
?column?
----------
@@ -487,54 +501,63 @@ SELECT $1 \bind 1 \g
-- \gx warning should be emitted in an aborted pipeline, with
-- pipeline still usable.
\startpipeline
-SELECT $1 \bind \g
+SELECT $1 \bind \sendpipeline
\flushrequest
\getresults
ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
SELECT $1 \bind 1 \gx
\gx not allowed in pipeline mode
\endpipeline
+-- \sendpipeline is not allowed outside of a pipeline
+\sendpipeline
+\sendpipeline not allowed outside of pipeline mode
+SELECT $1 \bind 1 \sendpipeline
+\sendpipeline not allowed outside of pipeline mode
+\reset
+-- \sendpipeline is not allowed if not preceded by \bind or \bind_named
+\startpipeline
+\sendpipeline
+\sendpipeline must be used after \bind or \bind_named
+SELECT 1 \sendpipeline
+\sendpipeline must be used after \bind or \bind_named
+\endpipeline
-- \gexec is not allowed, pipeline should still be usable.
\startpipeline
SELECT 'INSERT INTO psql_pipeline(a) SELECT generate_series(1, 10)' \parse 'insert_stmt'
\bind_named insert_stmt \gexec
\gexec not allowed in pipeline mode
-\bind_named insert_stmt \g
+\bind_named insert_stmt \sendpipeline
SELECT COUNT(*) FROM psql_pipeline \bind \g
+\g not allowed in pipeline mode
\endpipeline
?column?
------------------------------------------------------------
INSERT INTO psql_pipeline(a) SELECT generate_series(1, 10)
(1 row)
- count
--------
- 4
-(1 row)
-
-- After an error, pipeline is aborted and requires \syncpipeline to be
-- reusable.
\startpipeline
-SELECT $1 \bind \g
-SELECT $1 \bind 1 \g
+SELECT $1 \bind \sendpipeline
+SELECT $1 \bind 1 \sendpipeline
SELECT $1 \parse a
-\bind_named a 1 \g
+\bind_named a 1 \sendpipeline
\close a
\flushrequest
\getresults
ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
-- Pipeline is aborted.
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
SELECT $1 \parse a
-\bind_named a 1 \g
+\bind_named a 1 \sendpipeline
\close a
-- Sync allows pipeline to recover.
\syncpipeline
\getresults
Pipeline aborted, command did not run
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
SELECT $1 \parse a
-\bind_named a 1 \g
+\bind_named a 1 \sendpipeline
\close a
\flushrequest
\getresults
@@ -551,10 +574,10 @@ SELECT $1 \parse a
\endpipeline
-- In an aborted pipeline, \getresults 1 aborts commands one at a time.
\startpipeline
-SELECT $1 \bind \g
-SELECT $1 \bind 1 \g
+SELECT $1 \bind \sendpipeline
+SELECT $1 \bind 1 \sendpipeline
SELECT $1 \parse a
-\bind_named a 1 \g
+\bind_named a 1 \sendpipeline
\syncpipeline
\getresults 1
ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
@@ -569,11 +592,11 @@ Pipeline aborted, command did not run
-- Test chunked results with an aborted pipeline.
\startpipeline
\set FETCH_COUNT 10
-SELECT $1 \bind \g
+SELECT $1 \bind \sendpipeline
\flushrequest
\getresults
ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
-SELECT $1 \bind \g
+SELECT $1 \bind \sendpipeline
\endpipeline
fetching results in chunked mode failed
Pipeline aborted, command did not run
@@ -595,7 +618,7 @@ select 1;
-- Error messages accumulate and are repeated.
\startpipeline
-SELECT 1 \bind \g
+SELECT 1 \bind \sendpipeline
SELECT 1;
PQsendQuery not allowed in pipeline mode
SELECT 1;
@@ -616,12 +639,12 @@ PQsendQuery not allowed in pipeline mode
-- commit the implicit transaction block. The first command after a
-- sync will not be seen as belonging to a pipeline.
\startpipeline
-SET LOCAL statement_timeout='1h' \bind \g
-SHOW statement_timeout \bind \g
+SET LOCAL statement_timeout='1h' \bind \sendpipeline
+SHOW statement_timeout \bind \sendpipeline
\syncpipeline
-SHOW statement_timeout \bind \g
-SET LOCAL statement_timeout='2h' \bind \g
-SHOW statement_timeout \bind \g
+SHOW statement_timeout \bind \sendpipeline
+SET LOCAL statement_timeout='2h' \bind \sendpipeline
+SHOW statement_timeout \bind \sendpipeline
\endpipeline
WARNING: SET LOCAL can only be used in transaction blocks
statement_timeout
@@ -641,9 +664,9 @@ WARNING: SET LOCAL can only be used in transaction blocks
-- REINDEX CONCURRENTLY fails if not the first command in a pipeline.
\startpipeline
-SELECT $1 \bind 1 \g
-REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 1 \sendpipeline
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
?column?
----------
@@ -653,8 +676,8 @@ SELECT $1 \bind 2 \g
ERROR: REINDEX CONCURRENTLY cannot run inside a transaction block
-- REINDEX CONCURRENTLY works if it is the first command in a pipeline.
\startpipeline
-REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
-SELECT $1 \bind 2 \g
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
?column?
----------
@@ -663,25 +686,25 @@ SELECT $1 \bind 2 \g
-- Subtransactions are not allowed in a pipeline.
\startpipeline
-SAVEPOINT a \bind \g
-SELECT $1 \bind 1 \g
-ROLLBACK TO SAVEPOINT a \bind \g
-SELECT $1 \bind 2 \g
+SAVEPOINT a \bind \sendpipeline
+SELECT $1 \bind 1 \sendpipeline
+ROLLBACK TO SAVEPOINT a \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
ERROR: SAVEPOINT can only be used in transaction blocks
-- LOCK fails as the first command in a pipeline, as not seen in an
-- implicit transaction block.
\startpipeline
-LOCK psql_pipeline \bind \g
-SELECT $1 \bind 2 \g
+LOCK psql_pipeline \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
ERROR: LOCK TABLE can only be used in transaction blocks
-- LOCK succeeds as it is not the first command in a pipeline,
-- seen in an implicit transaction block.
\startpipeline
-SELECT $1 \bind 1 \g
-LOCK psql_pipeline \bind \g
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 1 \sendpipeline
+LOCK psql_pipeline \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
?column?
----------
@@ -695,12 +718,12 @@ SELECT $1 \bind 2 \g
-- VACUUM works as the first command in a pipeline.
\startpipeline
-VACUUM psql_pipeline \bind \g
+VACUUM psql_pipeline \bind \sendpipeline
\endpipeline
-- VACUUM fails when not the first command in a pipeline.
\startpipeline
-SELECT 1 \bind \g
-VACUUM psql_pipeline \bind \g
+SELECT 1 \bind \sendpipeline
+VACUUM psql_pipeline \bind \sendpipeline
\endpipeline
?column?
----------
@@ -710,9 +733,9 @@ VACUUM psql_pipeline \bind \g
ERROR: VACUUM cannot run inside a transaction block
-- VACUUM works after a \syncpipeline.
\startpipeline
-SELECT 1 \bind \g
+SELECT 1 \bind \sendpipeline
\syncpipeline
-VACUUM psql_pipeline \bind \g
+VACUUM psql_pipeline \bind \sendpipeline
\endpipeline
?column?
----------
diff --git a/src/test/regress/sql/psql.sql b/src/test/regress/sql/psql.sql
index 97d1be3aac3..b6ba947b812 100644
--- a/src/test/regress/sql/psql.sql
+++ b/src/test/regress/sql/psql.sql
@@ -1079,6 +1079,7 @@ select \if false \\ (bogus \else \\ 42 \endif \\ forty_two;
\sf whole_line
\sv whole_line
\startpipeline
+ \sendpipeline
\syncpipeline
\t arg1
\T arg1
diff --git a/src/test/regress/sql/psql_pipeline.sql b/src/test/regress/sql/psql_pipeline.sql
index 6517ebb71f8..256081dfd5f 100644
--- a/src/test/regress/sql/psql_pipeline.sql
+++ b/src/test/regress/sql/psql_pipeline.sql
@@ -6,23 +6,23 @@ CREATE TABLE psql_pipeline(a INTEGER PRIMARY KEY, s TEXT);
-- Single query
\startpipeline
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
-- Multiple queries
\startpipeline
-SELECT $1 \bind 'val1' \g
-SELECT $1, $2 \bind 'val2' 'val3' \g
-SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1 \bind 'val1' \sendpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
\endpipeline
-- Test \flush
\startpipeline
\flush
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\flush
-SELECT $1, $2 \bind 'val2' 'val3' \g
-SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
\endpipeline
-- Send multiple syncs
@@ -30,12 +30,12 @@ SELECT $1, $2 \bind 'val2' 'val3' \g
\echo :PIPELINE_COMMAND_COUNT
\echo :PIPELINE_SYNC_COUNT
\echo :PIPELINE_RESULT_COUNT
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\syncpipeline
\syncpipeline
-SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
\syncpipeline
-SELECT $1, $2 \bind 'val4' 'val5' \g
+SELECT $1, $2 \bind 'val4' 'val5' \sendpipeline
\echo :PIPELINE_COMMAND_COUNT
\echo :PIPELINE_SYNC_COUNT
\echo :PIPELINE_RESULT_COUNT
@@ -44,39 +44,39 @@ SELECT $1, $2 \bind 'val4' 'val5' \g
-- \startpipeline should not have any effect if already in a pipeline.
\startpipeline
\startpipeline
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
-- Convert an implicit transaction block to an explicit transaction block.
\startpipeline
-INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
-BEGIN \bind \g
-INSERT INTO psql_pipeline VALUES ($1) \bind 2 \g
-ROLLBACK \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \sendpipeline
+BEGIN \bind \sendpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 2 \sendpipeline
+ROLLBACK \bind \sendpipeline
\endpipeline
-- Multiple explicit transactions
\startpipeline
-BEGIN \bind \g
-INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
-ROLLBACK \bind \g
-BEGIN \bind \g
-INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
-COMMIT \bind \g
+BEGIN \bind \sendpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \sendpipeline
+ROLLBACK \bind \sendpipeline
+BEGIN \bind \sendpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \sendpipeline
+COMMIT \bind \sendpipeline
\endpipeline
-- COPY FROM STDIN
\startpipeline
-SELECT $1 \bind 'val1' \g
-COPY psql_pipeline FROM STDIN \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+COPY psql_pipeline FROM STDIN \bind \sendpipeline
\endpipeline
2 test2
\.
-- COPY FROM STDIN with \flushrequest + \getresults
\startpipeline
-SELECT $1 \bind 'val1' \g
-COPY psql_pipeline FROM STDIN \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+COPY psql_pipeline FROM STDIN \bind \sendpipeline
\flushrequest
\getresults
3 test3
@@ -85,8 +85,8 @@ COPY psql_pipeline FROM STDIN \bind \g
-- COPY FROM STDIN with \syncpipeline + \getresults
\startpipeline
-SELECT $1 \bind 'val1' \g
-COPY psql_pipeline FROM STDIN \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+COPY psql_pipeline FROM STDIN \bind \sendpipeline
\syncpipeline
\getresults
4 test4
@@ -95,22 +95,22 @@ COPY psql_pipeline FROM STDIN \bind \g
-- COPY TO STDOUT
\startpipeline
-SELECT $1 \bind 'val1' \g
-copy psql_pipeline TO STDOUT \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+copy psql_pipeline TO STDOUT \bind \sendpipeline
\endpipeline
-- COPY TO STDOUT with \flushrequest + \getresults
\startpipeline
-SELECT $1 \bind 'val1' \g
-copy psql_pipeline TO STDOUT \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+copy psql_pipeline TO STDOUT \bind \sendpipeline
\flushrequest
\getresults
\endpipeline
-- COPY TO STDOUT with \syncpipeline + \getresults
\startpipeline
-SELECT $1 \bind 'val1' \g
-copy psql_pipeline TO STDOUT \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+copy psql_pipeline TO STDOUT \bind \sendpipeline
\syncpipeline
\getresults
\endpipeline
@@ -120,22 +120,22 @@ copy psql_pipeline TO STDOUT \bind \g
SELECT $1 \parse ''
SELECT $1, $2 \parse ''
SELECT $2 \parse pipeline_1
-\bind_named '' 1 2 \g
-\bind_named pipeline_1 2 \g
+\bind_named '' 1 2 \sendpipeline
+\bind_named pipeline_1 2 \sendpipeline
\endpipeline
-- \getresults displays all results preceding a \flushrequest.
\startpipeline
-SELECT $1 \bind 1 \g
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 1 \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\flushrequest
\getresults
\endpipeline
-- \getresults displays all results preceding a \syncpipeline.
\startpipeline
-SELECT $1 \bind 1 \g
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 1 \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\syncpipeline
\getresults
\endpipeline
@@ -143,7 +143,7 @@ SELECT $1 \bind 2 \g
-- \getresults immediately returns if there is no result to fetch.
\startpipeline
\getresults
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\getresults
\flushrequest
\endpipeline
@@ -151,42 +151,42 @@ SELECT $1 \bind 2 \g
-- \getresults only fetches results preceding a \flushrequest.
\startpipeline
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\flushrequest
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\getresults
\endpipeline
-- \getresults only fetches results preceding a \syncpipeline.
\startpipeline
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\syncpipeline
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\getresults
\endpipeline
-- Use pipeline with chunked results for both \getresults and \endpipeline.
\startpipeline
\set FETCH_COUNT 10
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\flushrequest
\getresults
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
\unset FETCH_COUNT
-- \getresults with specific number of requested results.
\startpipeline
-SELECT $1 \bind 1 \g
-SELECT $1 \bind 2 \g
-SELECT $1 \bind 3 \g
+SELECT $1 \bind 1 \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
+SELECT $1 \bind 3 \sendpipeline
\echo :PIPELINE_SYNC_COUNT
\syncpipeline
\echo :PIPELINE_SYNC_COUNT
\echo :PIPELINE_RESULT_COUNT
\getresults 1
\echo :PIPELINE_RESULT_COUNT
-SELECT $1 \bind 4 \g
+SELECT $1 \bind 4 \sendpipeline
\getresults 3
\echo :PIPELINE_RESULT_COUNT
\endpipeline
@@ -195,7 +195,7 @@ SELECT $1 \bind 4 \g
\startpipeline
\syncpipeline
\syncpipeline
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
\flushrequest
\getresults 2
\getresults 1
@@ -203,9 +203,9 @@ SELECT $1 \bind 1 \g
-- \getresults 0 should get all the results.
\startpipeline
-SELECT $1 \bind 1 \g
-SELECT $1 \bind 2 \g
-SELECT $1 \bind 3 \g
+SELECT $1 \bind 1 \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
+SELECT $1 \bind 3 \sendpipeline
\syncpipeline
\getresults 0
\endpipeline
@@ -221,36 +221,36 @@ SELECT $1 \bind 3 \g
-- pipeline usable.
\startpipeline
SELECT 1;
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
-- After an aborted pipeline, commands after a \syncpipeline should be
-- displayed.
\startpipeline
-SELECT $1 \bind \g
+SELECT $1 \bind \sendpipeline
\syncpipeline
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
\endpipeline
-- For an incorrect number of parameters, the pipeline is aborted and
-- the following queries will not be executed.
\startpipeline
-SELECT \bind 'val1' \g
-SELECT $1 \bind 'val1' \g
+SELECT \bind 'val1' \sendpipeline
+SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
-- An explicit transaction with an error needs to be rollbacked after
-- the pipeline.
\startpipeline
-BEGIN \bind \g
-INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
-ROLLBACK \bind \g
+BEGIN \bind \sendpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \sendpipeline
+ROLLBACK \bind \sendpipeline
\endpipeline
ROLLBACK;
-- \watch sends a simple query, something not allowed within a pipeline.
\startpipeline
-SELECT \bind \g
+SELECT \bind \sendpipeline
\watch 1
\endpipeline
@@ -258,7 +258,7 @@ SELECT \bind \g
-- and the pipeline should still be usable.
\startpipeline
SELECT $1 \bind 1 \gdesc
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
\endpipeline
-- \gset is not allowed in a pipeline, pipeline should still be usable.
@@ -267,54 +267,72 @@ SELECT $1 as i, $2 as j \parse ''
SELECT $1 as k, $2 as l \parse 'second'
\bind_named '' 1 2 \gset
\bind_named second 1 2 \gset pref02_ \echo :pref02_i :pref02_j
-\bind_named '' 1 2 \g
+\bind_named '' 1 2 \sendpipeline
\endpipeline
--- \gx is not allowed, pipeline should still be usable.
+-- \g and \gx are not allowed, pipeline should still be usable.
\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \g filename
+SELECT $1 \bind 1 \g |cat
+SELECT $1 \bind 1 \g (format=unaligned tuples_only=on)
SELECT $1 \bind 1 \gx
+SELECT $1 \bind 1 \gx filename
+SELECT $1 \bind 1 \gx |cat
+SELECT $1 \bind 1 \gx (format=unaligned tuples_only=on)
\reset
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
\endpipeline
-- \gx warning should be emitted in an aborted pipeline, with
-- pipeline still usable.
\startpipeline
-SELECT $1 \bind \g
+SELECT $1 \bind \sendpipeline
\flushrequest
\getresults
SELECT $1 \bind 1 \gx
\endpipeline
+-- \sendpipeline is not allowed outside of a pipeline
+\sendpipeline
+SELECT $1 \bind 1 \sendpipeline
+\reset
+
+-- \sendpipeline is not allowed if not preceded by \bind or \bind_named
+\startpipeline
+\sendpipeline
+SELECT 1 \sendpipeline
+\endpipeline
+
-- \gexec is not allowed, pipeline should still be usable.
\startpipeline
SELECT 'INSERT INTO psql_pipeline(a) SELECT generate_series(1, 10)' \parse 'insert_stmt'
\bind_named insert_stmt \gexec
-\bind_named insert_stmt \g
+\bind_named insert_stmt \sendpipeline
SELECT COUNT(*) FROM psql_pipeline \bind \g
\endpipeline
-- After an error, pipeline is aborted and requires \syncpipeline to be
-- reusable.
\startpipeline
-SELECT $1 \bind \g
-SELECT $1 \bind 1 \g
+SELECT $1 \bind \sendpipeline
+SELECT $1 \bind 1 \sendpipeline
SELECT $1 \parse a
-\bind_named a 1 \g
+\bind_named a 1 \sendpipeline
\close a
\flushrequest
\getresults
-- Pipeline is aborted.
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
SELECT $1 \parse a
-\bind_named a 1 \g
+\bind_named a 1 \sendpipeline
\close a
-- Sync allows pipeline to recover.
\syncpipeline
\getresults
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
SELECT $1 \parse a
-\bind_named a 1 \g
+\bind_named a 1 \sendpipeline
\close a
\flushrequest
\getresults
@@ -322,10 +340,10 @@ SELECT $1 \parse a
-- In an aborted pipeline, \getresults 1 aborts commands one at a time.
\startpipeline
-SELECT $1 \bind \g
-SELECT $1 \bind 1 \g
+SELECT $1 \bind \sendpipeline
+SELECT $1 \bind 1 \sendpipeline
SELECT $1 \parse a
-\bind_named a 1 \g
+\bind_named a 1 \sendpipeline
\syncpipeline
\getresults 1
\getresults 1
@@ -337,10 +355,10 @@ SELECT $1 \parse a
-- Test chunked results with an aborted pipeline.
\startpipeline
\set FETCH_COUNT 10
-SELECT $1 \bind \g
+SELECT $1 \bind \sendpipeline
\flushrequest
\getresults
-SELECT $1 \bind \g
+SELECT $1 \bind \sendpipeline
\endpipeline
\unset FETCH_COUNT
@@ -356,7 +374,7 @@ select 1;
-- Error messages accumulate and are repeated.
\startpipeline
-SELECT 1 \bind \g
+SELECT 1 \bind \sendpipeline
SELECT 1;
SELECT 1;
\endpipeline
@@ -371,66 +389,66 @@ SELECT 1;
-- commit the implicit transaction block. The first command after a
-- sync will not be seen as belonging to a pipeline.
\startpipeline
-SET LOCAL statement_timeout='1h' \bind \g
-SHOW statement_timeout \bind \g
+SET LOCAL statement_timeout='1h' \bind \sendpipeline
+SHOW statement_timeout \bind \sendpipeline
\syncpipeline
-SHOW statement_timeout \bind \g
-SET LOCAL statement_timeout='2h' \bind \g
-SHOW statement_timeout \bind \g
+SHOW statement_timeout \bind \sendpipeline
+SET LOCAL statement_timeout='2h' \bind \sendpipeline
+SHOW statement_timeout \bind \sendpipeline
\endpipeline
-- REINDEX CONCURRENTLY fails if not the first command in a pipeline.
\startpipeline
-SELECT $1 \bind 1 \g
-REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 1 \sendpipeline
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
-- REINDEX CONCURRENTLY works if it is the first command in a pipeline.
\startpipeline
-REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
-SELECT $1 \bind 2 \g
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
-- Subtransactions are not allowed in a pipeline.
\startpipeline
-SAVEPOINT a \bind \g
-SELECT $1 \bind 1 \g
-ROLLBACK TO SAVEPOINT a \bind \g
-SELECT $1 \bind 2 \g
+SAVEPOINT a \bind \sendpipeline
+SELECT $1 \bind 1 \sendpipeline
+ROLLBACK TO SAVEPOINT a \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
-- LOCK fails as the first command in a pipeline, as not seen in an
-- implicit transaction block.
\startpipeline
-LOCK psql_pipeline \bind \g
-SELECT $1 \bind 2 \g
+LOCK psql_pipeline \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
-- LOCK succeeds as it is not the first command in a pipeline,
-- seen in an implicit transaction block.
\startpipeline
-SELECT $1 \bind 1 \g
-LOCK psql_pipeline \bind \g
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 1 \sendpipeline
+LOCK psql_pipeline \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
-- VACUUM works as the first command in a pipeline.
\startpipeline
-VACUUM psql_pipeline \bind \g
+VACUUM psql_pipeline \bind \sendpipeline
\endpipeline
-- VACUUM fails when not the first command in a pipeline.
\startpipeline
-SELECT 1 \bind \g
-VACUUM psql_pipeline \bind \g
+SELECT 1 \bind \sendpipeline
+VACUUM psql_pipeline \bind \sendpipeline
\endpipeline
-- VACUUM works after a \syncpipeline.
\startpipeline
-SELECT 1 \bind \g
+SELECT 1 \bind \sendpipeline
\syncpipeline
-VACUUM psql_pipeline \bind \g
+VACUUM psql_pipeline \bind \sendpipeline
\endpipeline
-- Clean up
--
2.39.5 (Apple Git-154)
[application/octet-stream] v02-0002-psql-Allow-to-add-queries-in-an-ongoing-pipeline.patch (7.5K, ../../CAO6_Xqq4JEHRaGyC=r+dpi0Ejbp2P3aUo-KfR2NuipCvE-o_ZA@mail.gmail.com/3-v02-0002-psql-Allow-to-add-queries-in-an-ongoing-pipeline.patch)
download | inline diff:
From 2ebe091c9fcc5cd35bbbc02ece90645539330ebd Mon Sep 17 00:00:00 2001
From: Anthonin Bonnefoy <[email protected]>
Date: Wed, 5 Mar 2025 14:55:33 +0100
Subject: psql: Allow ';' to add queries in an ongoing pipeline
Currently, the only way to pipe queries in an ongoing pipeline is to
leverage psql meta-commands to create extended queries such as \bind,
\parse or \bind_named. This prevents using psql's pipeline on existing
scripts as it would require to convert all queries to use those
meta-commands.
This patch modifies ';' behaviour within an active pipeline and send all
queries as extended queries, allowing them to be piped in a pipeline.
---
doc/src/sgml/ref/psql-ref.sgml | 21 +++++----
src/bin/psql/command.c | 7 +++
src/bin/psql/common.c | 10 +++-
src/test/regress/expected/psql_pipeline.out | 52 +++++++++++++--------
src/test/regress/sql/psql_pipeline.sql | 24 ++++++----
5 files changed, 77 insertions(+), 37 deletions(-)
diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml
index cddf6e07531..2763486e268 100644
--- a/doc/src/sgml/ref/psql-ref.sgml
+++ b/doc/src/sgml/ref/psql-ref.sgml
@@ -3698,14 +3698,19 @@ testdb=> <userinput>\setenv LESS -imx4F</userinput>
</para>
<para>
- Pipeline mode requires the use of the extended query protocol. All
- queries need to be sent using the meta-commands
- <literal>\bind</literal>, <literal>\bind_named</literal>,
- <literal>\close</literal> or <literal>\parse</literal>. While a
- pipeline is ongoing, <literal>\sendpipeline</literal> will append the
- current query buffer to the pipeline. Other meta-commands like
- <literal>\g</literal>, <literal>\gx</literal> or <literal>\gdesc</literal>
- are not allowed in pipeline mode.
+ Pipeline mode requires the use of the extended query protocol. Queries
+ can be sent using the meta-commands <literal>\bind</literal>,
+ <literal>\bind_named</literal>, <literal>\close</literal> or
+ <literal>\parse</literal>. While a pipeline is ongoing,
+ <literal>\sendpipeline</literal> will append the current query
+ buffer to the pipeline. Other meta-commands like <literal>\g</literal>,
+ <literal>\gx</literal> or <literal>\gdesc</literal> are not allowed
+ in pipeline mode.
+ </para>
+
+ <para>
+ Queries can also be sent using <literal>;</literal>. While a pipeline is
+ ongoing, they will automatically be sent using extended query protocol.
</para>
<para>
diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c
index 7670c20b29e..3e7eb086367 100644
--- a/src/bin/psql/command.c
+++ b/src/bin/psql/command.c
@@ -3282,6 +3282,13 @@ exec_command_watch(PsqlScanState scan_state, bool active_branch,
int iter = 0;
int min_rows = 0;
+ if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pg_log_error("\\watch not allowed in pipeline mode");
+ clean_extended_state();
+ success = false;
+ }
+
/*
* Parse arguments. We allow either an unlabeled interval or
* "name=value", where name is from the set ('i', 'interval', 'c',
diff --git a/src/bin/psql/common.c b/src/bin/psql/common.c
index ed340a466f9..5249336bcf2 100644
--- a/src/bin/psql/common.c
+++ b/src/bin/psql/common.c
@@ -1668,7 +1668,15 @@ ExecQueryAndProcessResults(const char *query,
}
break;
case PSQL_SEND_QUERY:
- success = PQsendQuery(pset.db, query);
+ if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ success = PQsendQueryParams(pset.db, query,
+ 0, NULL, NULL, NULL, NULL, 0);
+ if (success)
+ pset.piped_commands++;
+ }
+ else
+ success = PQsendQuery(pset.db, query);
break;
}
diff --git a/src/test/regress/expected/psql_pipeline.out b/src/test/regress/expected/psql_pipeline.out
index 53f2edfd986..f05429b36ac 100644
--- a/src/test/regress/expected/psql_pipeline.out
+++ b/src/test/regress/expected/psql_pipeline.out
@@ -387,24 +387,33 @@ SELECT $1 \bind 3 \sendpipeline
(1 row)
\endpipeline
+-- Queries can be pipelined ';'. They will be sent using extended protocol.
+\startpipeline
+SELECT 1;
+SELECT $1 \bind 'val1' \sendpipeline
+SELECT 2;
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ val1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
--
-- Pipeline errors
--
-- \endpipeline outside of pipeline should fail
\endpipeline
cannot send pipeline when not in pipeline mode
--- Query using simple protocol should not be sent and should leave the
--- pipeline usable.
-\startpipeline
-SELECT 1;
-PQsendQuery not allowed in pipeline mode
-SELECT $1 \bind 'val1' \sendpipeline
-\endpipeline
- ?column?
-----------
- val1
-(1 row)
-
-- After an aborted pipeline, commands after a \syncpipeline should be
-- displayed.
\startpipeline
@@ -425,6 +434,12 @@ SELECT \bind 'val1' \sendpipeline
SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
ERROR: bind message supplies 1 parameters, but prepared statement "" requires 0
+-- Using ';' with a parameter will trigger an incorrect parameter errors.
+\startpipeline
+SELECT $1;
+SELECT 1;
+\endpipeline
+ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
-- An explicit transaction with an error needs to be rollbacked after
-- the pipeline.
\startpipeline
@@ -439,8 +454,7 @@ ROLLBACK;
\startpipeline
SELECT \bind \sendpipeline
\watch 1
-PQsendQuery not allowed in pipeline mode
-
+\watch not allowed in pipeline mode
\endpipeline
--
(1 row)
@@ -619,11 +633,11 @@ select 1;
-- Error messages accumulate and are repeated.
\startpipeline
SELECT 1 \bind \sendpipeline
-SELECT 1;
-PQsendQuery not allowed in pipeline mode
-SELECT 1;
-PQsendQuery not allowed in pipeline mode
-PQsendQuery not allowed in pipeline mode
+\gdesc
+synchronous command execution functions are not allowed in pipeline mode
+\gdesc
+synchronous command execution functions are not allowed in pipeline mode
+synchronous command execution functions are not allowed in pipeline mode
\endpipeline
?column?
----------
diff --git a/src/test/regress/sql/psql_pipeline.sql b/src/test/regress/sql/psql_pipeline.sql
index 256081dfd5f..76c0ece01af 100644
--- a/src/test/regress/sql/psql_pipeline.sql
+++ b/src/test/regress/sql/psql_pipeline.sql
@@ -210,6 +210,13 @@ SELECT $1 \bind 3 \sendpipeline
\getresults 0
\endpipeline
+-- Queries can be pipelined ';'. They will be sent using extended protocol.
+\startpipeline
+SELECT 1;
+SELECT $1 \bind 'val1' \sendpipeline
+SELECT 2;
+\endpipeline
+
--
-- Pipeline errors
--
@@ -217,13 +224,6 @@ SELECT $1 \bind 3 \sendpipeline
-- \endpipeline outside of pipeline should fail
\endpipeline
--- Query using simple protocol should not be sent and should leave the
--- pipeline usable.
-\startpipeline
-SELECT 1;
-SELECT $1 \bind 'val1' \sendpipeline
-\endpipeline
-
-- After an aborted pipeline, commands after a \syncpipeline should be
-- displayed.
\startpipeline
@@ -239,6 +239,12 @@ SELECT \bind 'val1' \sendpipeline
SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
+-- Using ';' with a parameter will trigger an incorrect parameter errors.
+\startpipeline
+SELECT $1;
+SELECT 1;
+\endpipeline
+
-- An explicit transaction with an error needs to be rollbacked after
-- the pipeline.
\startpipeline
@@ -375,8 +381,8 @@ select 1;
-- Error messages accumulate and are repeated.
\startpipeline
SELECT 1 \bind \sendpipeline
-SELECT 1;
-SELECT 1;
+\gdesc
+\gdesc
\endpipeline
--
--
2.39.5 (Apple Git-154)
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-17 10:19 Michael Paquier <[email protected]>
parent: Anthonin Bonnefoy <[email protected]>
2 siblings, 0 replies; 55+ messages in thread
From: Michael Paquier @ 2025-03-17 10:19 UTC (permalink / raw)
To: Anthonin Bonnefoy <[email protected]>; +Cc: Daniel Verite <[email protected]>; Jelte Fennema-Nio <[email protected]>; pgsql-hackers
On Mon, Mar 17, 2025 at 10:50:50AM +0100, Anthonin Bonnefoy wrote:
> 0001: This introduces the \sendpipeline meta-command and forbid \g in
> a pipeline. This is to fix the formatting options of \g that are not
> supported in a pipeline.
>
> 0002: Allows ';' to send a query using extended protocol when within a
> pipeline by using PQsendQueryParams with 0 parameters. It is not
> possible to send parameters with extended protocol this way and
> everything will be propagated through the query string, similar to a
> simple query.
Thanks for sending a new patch set. I was planning to look at the
situation tomorrow, and you have beaten me to it.
The split makes sense, and I'm OK with 0001. 0002 is going to require
a much closer lookup.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-18 00:50 Michael Paquier <[email protected]>
parent: Anthonin Bonnefoy <[email protected]>
2 siblings, 1 reply; 55+ messages in thread
From: Michael Paquier @ 2025-03-18 00:50 UTC (permalink / raw)
To: Anthonin Bonnefoy <[email protected]>; +Cc: Daniel Verite <[email protected]>; Jelte Fennema-Nio <[email protected]>; pgsql-hackers
On Mon, Mar 17, 2025 at 10:50:50AM +0100, Anthonin Bonnefoy wrote:
> 0001: This introduces the \sendpipeline meta-command and forbid \g in
> a pipeline. This is to fix the formatting options of \g that are not
> supported in a pipeline.
- count
--------
- 4
-(1 row)
This removal done in the regression tests was not intentional.
I have done some reordering of the code around the new meta-command so
as things are ordered alphabetically, and applied the result.
> 0002: Allows ';' to send a query using extended protocol when within a
> pipeline by using PQsendQueryParams with 0 parameters. It is not
> possible to send parameters with extended protocol this way and
> everything will be propagated through the query string, similar to a
> simple query.
I like the simplicity of what you are doing here, relying on
PSQL_SEND_QUERY being the default so as we use PQsendQueryParams()
with no parameters rather than PQsendQuery() when the pipeline mode is
not off.
How about adding a check on PIPELINE_COMMAND_COUNT when sending a
query through this path? Should we check for more scenarios with
syncs and flushes as well when sending these queries?
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-18 08:55 Anthonin Bonnefoy <[email protected]>
parent: Michael Paquier <[email protected]>
0 siblings, 2 replies; 55+ messages in thread
From: Anthonin Bonnefoy @ 2025-03-18 08:55 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Daniel Verite <[email protected]>; Jelte Fennema-Nio <[email protected]>; pgsql-hackers
On Tue, Mar 18, 2025 at 1:50 AM Michael Paquier <[email protected]> wrote:
> - count
> --------
> - 4
> -(1 row)
>
> This removal done in the regression tests was not intentional.
Yes, thanks for fixing that.
> How about adding a check on PIPELINE_COMMAND_COUNT when sending a
> query through this path? Should we check for more scenarios with
> syncs and flushes as well when sending these queries?
I've added additional tests when piping queries with ';':
- I've reused the same scenario with \sendpipeline: single query,
multiple queries, flushes, syncs, using COPY...
- Using ';' will replace the unnamed prepared statement. It's a bit
different from expected as a simple query will delete the unnamed
prepared statement.
- Sending an extended query prepared with \bind using a ';' on a
newline, though this is not specific to pipelining. The scanned
semicolon triggers the call to SendQuery, processing the buffered
extended query. It's a bit unusual but that's the current behaviour.
Attachments:
[application/octet-stream] v03-0001-psql-Allow-to-add-queries-in-an-ongoing-pipeline.patch (13.5K, ../../CAO6_Xqo=fNh0KCRZG7T18OZfjiabeL-t4C6mkXsz-3dAZ1YwTA@mail.gmail.com/2-v03-0001-psql-Allow-to-add-queries-in-an-ongoing-pipeline.patch)
download | inline diff:
From ae9ce7e44c1e2502429920c5d305e1ffcd30b1a5 Mon Sep 17 00:00:00 2001
From: Anthonin Bonnefoy <[email protected]>
Date: Wed, 5 Mar 2025 14:55:33 +0100
Subject: psql: Allow ';' to add queries in an ongoing pipeline
Currently, the only way to pipe queries in an ongoing pipeline is to
leverage psql meta-commands to create extended queries such as \bind,
\parse or \bind_named. This prevents using psql's pipeline on existing
scripts as it would require to convert all queries to use those
meta-commands.
This patch modifies ';' behaviour within an active pipeline and send all
queries as extended queries, allowing them to be piped in a pipeline.
---
doc/src/sgml/ref/psql-ref.sgml | 21 +-
src/bin/psql/command.c | 7 +
src/bin/psql/common.c | 10 +-
src/test/regress/expected/psql_pipeline.out | 292 ++++++++++++++++++--
src/test/regress/sql/psql_pipeline.sql | 139 +++++++++-
5 files changed, 429 insertions(+), 40 deletions(-)
diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml
index cddf6e07531..2763486e268 100644
--- a/doc/src/sgml/ref/psql-ref.sgml
+++ b/doc/src/sgml/ref/psql-ref.sgml
@@ -3698,14 +3698,19 @@ testdb=> <userinput>\setenv LESS -imx4F</userinput>
</para>
<para>
- Pipeline mode requires the use of the extended query protocol. All
- queries need to be sent using the meta-commands
- <literal>\bind</literal>, <literal>\bind_named</literal>,
- <literal>\close</literal> or <literal>\parse</literal>. While a
- pipeline is ongoing, <literal>\sendpipeline</literal> will append the
- current query buffer to the pipeline. Other meta-commands like
- <literal>\g</literal>, <literal>\gx</literal> or <literal>\gdesc</literal>
- are not allowed in pipeline mode.
+ Pipeline mode requires the use of the extended query protocol. Queries
+ can be sent using the meta-commands <literal>\bind</literal>,
+ <literal>\bind_named</literal>, <literal>\close</literal> or
+ <literal>\parse</literal>. While a pipeline is ongoing,
+ <literal>\sendpipeline</literal> will append the current query
+ buffer to the pipeline. Other meta-commands like <literal>\g</literal>,
+ <literal>\gx</literal> or <literal>\gdesc</literal> are not allowed
+ in pipeline mode.
+ </para>
+
+ <para>
+ Queries can also be sent using <literal>;</literal>. While a pipeline is
+ ongoing, they will automatically be sent using extended query protocol.
</para>
<para>
diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c
index a87ff7e4597..bbe337780ff 100644
--- a/src/bin/psql/command.c
+++ b/src/bin/psql/command.c
@@ -3282,6 +3282,13 @@ exec_command_watch(PsqlScanState scan_state, bool active_branch,
int iter = 0;
int min_rows = 0;
+ if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pg_log_error("\\watch not allowed in pipeline mode");
+ clean_extended_state();
+ success = false;
+ }
+
/*
* Parse arguments. We allow either an unlabeled interval or
* "name=value", where name is from the set ('i', 'interval', 'c',
diff --git a/src/bin/psql/common.c b/src/bin/psql/common.c
index ed340a466f9..5249336bcf2 100644
--- a/src/bin/psql/common.c
+++ b/src/bin/psql/common.c
@@ -1668,7 +1668,15 @@ ExecQueryAndProcessResults(const char *query,
}
break;
case PSQL_SEND_QUERY:
- success = PQsendQuery(pset.db, query);
+ if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ success = PQsendQueryParams(pset.db, query,
+ 0, NULL, NULL, NULL, NULL, 0);
+ if (success)
+ pset.piped_commands++;
+ }
+ else
+ success = PQsendQuery(pset.db, query);
break;
}
diff --git a/src/test/regress/expected/psql_pipeline.out b/src/test/regress/expected/psql_pipeline.out
index 68e3c19ea05..7dddf26a9fd 100644
--- a/src/test/regress/expected/psql_pipeline.out
+++ b/src/test/regress/expected/psql_pipeline.out
@@ -386,6 +386,262 @@ SELECT $1 \bind 3 \sendpipeline
3
(1 row)
+\endpipeline
+--
+-- Tests pipelining queries with ';'
+--
+-- Single query sent with ';'
+\startpipeline
+SELECT 1;
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+-- Multiple queries sent with ';'
+\startpipeline
+SELECT 1;
+SELECT 2;
+SELECT 3;
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+ ?column?
+----------
+ 3
+(1 row)
+
+-- Multiple queries on the same line can be piped with ';'
+\startpipeline
+SELECT 1; SELECT 2; SELECT 3
+;
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+ ?column?
+----------
+ 3
+(1 row)
+
+-- Test \flush with queries piped with ';'
+\startpipeline
+\flush
+SELECT 1;
+\flush
+SELECT 2;
+SELECT 3;
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+ ?column?
+----------
+ 3
+(1 row)
+
+-- Send multiple syncs with queries piped with ';'
+\startpipeline
+\echo :PIPELINE_COMMAND_COUNT
+0
+\echo :PIPELINE_SYNC_COUNT
+0
+\echo :PIPELINE_RESULT_COUNT
+0
+SELECT 1;
+\syncpipeline
+\syncpipeline
+SELECT 2;
+\syncpipeline
+SELECT 3;
+\echo :PIPELINE_COMMAND_COUNT
+1
+\echo :PIPELINE_SYNC_COUNT
+3
+\echo :PIPELINE_RESULT_COUNT
+2
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+ ?column?
+----------
+ 3
+(1 row)
+
+-- Mix queries piped with ';' and \sendpipeline
+\startpipeline
+SELECT 1;
+SELECT $1 \bind 'val1' \sendpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
+SELECT 2;
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ val1
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val2 | val3
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+-- Piping a query with ';' will replace the unnamed prepared statement
+\startpipeline
+SELECT $1 \parse ''
+SELECT 1;
+\bind_named ''
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+-- An extended query can be piped by a ';' after a newline
+\startpipeline
+SELECT $1 \bind 1
+;
+SELECT 2;
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+-- COPY FROM STDIN, using ';'
+\startpipeline
+SELECT 'val1';
+COPY psql_pipeline FROM STDIN;
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+-- COPY FROM STDIN with \flushrequest + \getresults, using ';'
+\startpipeline
+SELECT 'val1';
+COPY psql_pipeline FROM STDIN;
+\flushrequest
+\getresults
+ ?column?
+----------
+ val1
+(1 row)
+
+message type 0x5a arrived from server while idle
+\endpipeline
+-- COPY FROM STDIN with \syncpipeline + \getresults, using ';'
+\startpipeline
+SELECT 'val1';
+COPY psql_pipeline FROM STDIN;
+\syncpipeline
+\getresults
+ ?column?
+----------
+ val1
+(1 row)
+
+\endpipeline
+-- COPY TO STDOUT, using ';'
+\startpipeline
+SELECT 'val1';
+copy psql_pipeline TO STDOUT;
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+1 \N
+2 test2
+3 test3
+4 test4
+20 test2
+30 test3
+40 test4
+-- COPY TO STDOUT with \flushrequest + \getresults, using ';'
+\startpipeline
+SELECT 'val1';
+copy psql_pipeline TO STDOUT;
+\flushrequest
+\getresults
+ ?column?
+----------
+ val1
+(1 row)
+
+1 \N
+2 test2
+3 test3
+4 test4
+20 test2
+30 test3
+40 test4
+\endpipeline
+-- COPY TO STDOUT with \syncpipeline + \getresults, using ';'
+\startpipeline
+SELECT 'val1';
+copy psql_pipeline TO STDOUT;
+\syncpipeline
+\getresults
+ ?column?
+----------
+ val1
+(1 row)
+
+1 \N
+2 test2
+3 test3
+4 test4
+20 test2
+30 test3
+40 test4
\endpipeline
--
-- Pipeline errors
@@ -393,18 +649,6 @@ SELECT $1 \bind 3 \sendpipeline
-- \endpipeline outside of pipeline should fail
\endpipeline
cannot send pipeline when not in pipeline mode
--- Query using simple protocol should not be sent and should leave the
--- pipeline usable.
-\startpipeline
-SELECT 1;
-PQsendQuery not allowed in pipeline mode
-SELECT $1 \bind 'val1' \sendpipeline
-\endpipeline
- ?column?
-----------
- val1
-(1 row)
-
-- After an aborted pipeline, commands after a \syncpipeline should be
-- displayed.
\startpipeline
@@ -425,6 +669,13 @@ SELECT \bind 'val1' \sendpipeline
SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
ERROR: bind message supplies 1 parameters, but prepared statement "" requires 0
+-- Using ';' with a parameter will trigger an incorrect parameter error and
+-- abort the pipeline
+\startpipeline
+SELECT $1;
+SELECT 1;
+\endpipeline
+ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
-- An explicit transaction with an error needs to be rollbacked after
-- the pipeline.
\startpipeline
@@ -435,12 +686,11 @@ ROLLBACK \bind \sendpipeline
ERROR: duplicate key value violates unique constraint "psql_pipeline_pkey"
DETAIL: Key (a)=(1) already exists.
ROLLBACK;
--- \watch sends a simple query, something not allowed within a pipeline.
+-- \watch is not allowed within a pipeline.
\startpipeline
SELECT \bind \sendpipeline
\watch 1
-PQsendQuery not allowed in pipeline mode
-
+\watch not allowed in pipeline mode
\endpipeline
--
(1 row)
@@ -530,7 +780,7 @@ SELECT COUNT(*) FROM psql_pipeline \bind \sendpipeline
count
-------
- 4
+ 7
(1 row)
-- After an error, pipeline is aborted and requires \syncpipeline to be
@@ -617,11 +867,11 @@ select 1;
-- Error messages accumulate and are repeated.
\startpipeline
SELECT 1 \bind \sendpipeline
-SELECT 1;
-PQsendQuery not allowed in pipeline mode
-SELECT 1;
-PQsendQuery not allowed in pipeline mode
-PQsendQuery not allowed in pipeline mode
+\gdesc
+synchronous command execution functions are not allowed in pipeline mode
+\gdesc
+synchronous command execution functions are not allowed in pipeline mode
+synchronous command execution functions are not allowed in pipeline mode
\endpipeline
?column?
----------
diff --git a/src/test/regress/sql/psql_pipeline.sql b/src/test/regress/sql/psql_pipeline.sql
index e4d7e614af3..6f76adcba82 100644
--- a/src/test/regress/sql/psql_pipeline.sql
+++ b/src/test/regress/sql/psql_pipeline.sql
@@ -210,6 +210,125 @@ SELECT $1 \bind 3 \sendpipeline
\getresults 0
\endpipeline
+--
+-- Tests pipelining queries with ';'
+--
+
+-- Single query sent with ';'
+\startpipeline
+SELECT 1;
+\endpipeline
+
+-- Multiple queries sent with ';'
+\startpipeline
+SELECT 1;
+SELECT 2;
+SELECT 3;
+\endpipeline
+
+-- Multiple queries on the same line can be piped with ';'
+\startpipeline
+SELECT 1; SELECT 2; SELECT 3
+;
+\endpipeline
+
+-- Test \flush with queries piped with ';'
+\startpipeline
+\flush
+SELECT 1;
+\flush
+SELECT 2;
+SELECT 3;
+\endpipeline
+
+-- Send multiple syncs with queries piped with ';'
+\startpipeline
+\echo :PIPELINE_COMMAND_COUNT
+\echo :PIPELINE_SYNC_COUNT
+\echo :PIPELINE_RESULT_COUNT
+SELECT 1;
+\syncpipeline
+\syncpipeline
+SELECT 2;
+\syncpipeline
+SELECT 3;
+\echo :PIPELINE_COMMAND_COUNT
+\echo :PIPELINE_SYNC_COUNT
+\echo :PIPELINE_RESULT_COUNT
+\endpipeline
+
+-- Mix queries piped with ';' and \sendpipeline
+\startpipeline
+SELECT 1;
+SELECT $1 \bind 'val1' \sendpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
+SELECT 2;
+\endpipeline
+
+-- Piping a query with ';' will replace the unnamed prepared statement
+\startpipeline
+SELECT $1 \parse ''
+SELECT 1;
+\bind_named ''
+\endpipeline
+
+-- An extended query can be piped by a ';' after a newline
+\startpipeline
+SELECT $1 \bind 1
+;
+SELECT 2;
+\endpipeline
+
+-- COPY FROM STDIN, using ';'
+\startpipeline
+SELECT 'val1';
+COPY psql_pipeline FROM STDIN;
+\endpipeline
+20 test2
+\.
+
+-- COPY FROM STDIN with \flushrequest + \getresults, using ';'
+\startpipeline
+SELECT 'val1';
+COPY psql_pipeline FROM STDIN;
+\flushrequest
+\getresults
+30 test3
+\.
+\endpipeline
+
+-- COPY FROM STDIN with \syncpipeline + \getresults, using ';'
+\startpipeline
+SELECT 'val1';
+COPY psql_pipeline FROM STDIN;
+\syncpipeline
+\getresults
+40 test4
+\.
+\endpipeline
+
+-- COPY TO STDOUT, using ';'
+\startpipeline
+SELECT 'val1';
+copy psql_pipeline TO STDOUT;
+\endpipeline
+
+-- COPY TO STDOUT with \flushrequest + \getresults, using ';'
+\startpipeline
+SELECT 'val1';
+copy psql_pipeline TO STDOUT;
+\flushrequest
+\getresults
+\endpipeline
+
+-- COPY TO STDOUT with \syncpipeline + \getresults, using ';'
+\startpipeline
+SELECT 'val1';
+copy psql_pipeline TO STDOUT;
+\syncpipeline
+\getresults
+\endpipeline
+
--
-- Pipeline errors
--
@@ -217,13 +336,6 @@ SELECT $1 \bind 3 \sendpipeline
-- \endpipeline outside of pipeline should fail
\endpipeline
--- Query using simple protocol should not be sent and should leave the
--- pipeline usable.
-\startpipeline
-SELECT 1;
-SELECT $1 \bind 'val1' \sendpipeline
-\endpipeline
-
-- After an aborted pipeline, commands after a \syncpipeline should be
-- displayed.
\startpipeline
@@ -239,6 +351,13 @@ SELECT \bind 'val1' \sendpipeline
SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
+-- Using ';' with a parameter will trigger an incorrect parameter error and
+-- abort the pipeline
+\startpipeline
+SELECT $1;
+SELECT 1;
+\endpipeline
+
-- An explicit transaction with an error needs to be rollbacked after
-- the pipeline.
\startpipeline
@@ -248,7 +367,7 @@ ROLLBACK \bind \sendpipeline
\endpipeline
ROLLBACK;
--- \watch sends a simple query, something not allowed within a pipeline.
+-- \watch is not allowed within a pipeline.
\startpipeline
SELECT \bind \sendpipeline
\watch 1
@@ -372,8 +491,8 @@ select 1;
-- Error messages accumulate and are repeated.
\startpipeline
SELECT 1 \bind \sendpipeline
-SELECT 1;
-SELECT 1;
+\gdesc
+\gdesc
\endpipeline
--
--
2.39.5 (Apple Git-154)
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-18 09:27 Jelte Fennema-Nio <[email protected]>
parent: Anthonin Bonnefoy <[email protected]>
1 sibling, 1 reply; 55+ messages in thread
From: Jelte Fennema-Nio @ 2025-03-18 09:27 UTC (permalink / raw)
To: Anthonin Bonnefoy <[email protected]>; +Cc: Michael Paquier <[email protected]>; Daniel Verite <[email protected]>; pgsql-hackers
On Tue, 18 Mar 2025 at 09:55, Anthonin Bonnefoy
<[email protected]> wrote:
> I've added additional tests when piping queries with ';':
> - I've reused the same scenario with \sendpipeline: single query,
> multiple queries, flushes, syncs, using COPY...
> - Using ';' will replace the unnamed prepared statement. It's a bit
> different from expected as a simple query will delete the unnamed
> prepared statement.
> - Sending an extended query prepared with \bind using a ';' on a
> newline, though this is not specific to pipelining. The scanned
> semicolon triggers the call to SendQuery, processing the buffered
> extended query. It's a bit unusual but that's the current behaviour.
One thing that comes to mind that I think would be quite useful and
pretty easy to implement if we have this functionality within a
pipeline: An \extended command. That puts psql in "extended protocol
mode" (without enabling pipelining). In "extended protocol mode" all
queries would automatically be sent using PQsendQueryParams. That
would remove the need to use \bind anymore outside of a pipeline
either.
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-18 09:36 Daniel Verite <[email protected]>
parent: Anthonin Bonnefoy <[email protected]>
2 siblings, 1 reply; 55+ messages in thread
From: Daniel Verite @ 2025-03-18 09:36 UTC (permalink / raw)
To: Anthonin Bonnefoy <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; Michael Paquier <[email protected]>; pgsql-hackers
Anthonin Bonnefoy wrote:
> 0002: Allows ';' to send a query using extended protocol when within a
> pipeline by using PQsendQueryParams
It's a nice improvement!
> with 0 parameters. It is not
> possible to send parameters with extended protocol this way and
> everything will be propagated through the query string, similar to a
> simple query.
It's actually possible to use parameters
\startpipeline
\bind 'foo'
select $1;
\endpipeline
?column?
----------
foo
(1 row)
I suspect there's a misunderstanding that \bind can only be placed
after the query, because it's always written like that in the regression
tests, and in the documentation.
But that's just a notational preference.
Best regards,
--
Daniel Vérité
https://postgresql.verite.pro/
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-18 23:56 Michael Paquier <[email protected]>
parent: Daniel Verite <[email protected]>
0 siblings, 0 replies; 55+ messages in thread
From: Michael Paquier @ 2025-03-18 23:56 UTC (permalink / raw)
To: Daniel Verite <[email protected]>; +Cc: Anthonin Bonnefoy <[email protected]>; Jelte Fennema-Nio <[email protected]>; pgsql-hackers
On Tue, Mar 18, 2025 at 10:36:28AM +0100, Daniel Verite wrote:
> It's actually possible to use parameters
>
> \startpipeline
> \bind 'foo'
> select $1;
> \endpipeline
>
> ?column?
> ----------
> foo
> (1 row)
>
> I suspect there's a misunderstanding that \bind can only be placed
> after the query, because it's always written like that in the regression
> tests, and in the documentation.
> But that's just a notational preference.
Nice trick, unrelated to pipelines. I don't think that we have
anything testing this specific pattern for \bind. At quick glance all
our test use \bind at the end of a query string. Perhaps we should?
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-19 02:28 Michael Paquier <[email protected]>
parent: Jelte Fennema-Nio <[email protected]>
0 siblings, 1 reply; 55+ messages in thread
From: Michael Paquier @ 2025-03-19 02:28 UTC (permalink / raw)
To: Jelte Fennema-Nio <[email protected]>; +Cc: Anthonin Bonnefoy <[email protected]>; Daniel Verite <[email protected]>; pgsql-hackers
On Tue, Mar 18, 2025 at 10:27:38AM +0100, Jelte Fennema-Nio wrote:
> One thing that comes to mind that I think would be quite useful and
> pretty easy to implement if we have this functionality within a
> pipeline: An \extended command. That puts psql in "extended protocol
> mode" (without enabling pipelining). In "extended protocol mode" all
> queries would automatically be sent using PQsendQueryParams. That
> would remove the need to use \bind anymore outside of a pipeline
> either.
How does that help when passing parameter values? \bind is here to be
able to pass down parameter values to queries that are prepared, so we
cannot bypass it as the parameter values need to be passed to the
\bind meta-command itself.
Perhaps an \extended command that behaves outside a pipeline makes
sense to force the use of queries without parameters to use the
extended mode, but I cannot get much excited about the concept knowing
all the meta-commands we have now (not talking about the pipeline
part, which is different, as we can treat queries in batches).
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-19 04:49 Michael Paquier <[email protected]>
parent: Anthonin Bonnefoy <[email protected]>
1 sibling, 0 replies; 55+ messages in thread
From: Michael Paquier @ 2025-03-19 04:49 UTC (permalink / raw)
To: Anthonin Bonnefoy <[email protected]>; +Cc: Daniel Verite <[email protected]>; Jelte Fennema-Nio <[email protected]>; pgsql-hackers
On Tue, Mar 18, 2025 at 09:55:21AM +0100, Anthonin Bonnefoy wrote:
> I've added additional tests when piping queries with ';':
> - I've reused the same scenario with \sendpipeline: single query,
> multiple queries, flushes, syncs, using COPY...
> - Using ';' will replace the unnamed prepared statement. It's a bit
> different from expected as a simple query will delete the unnamed
> prepared statement.
> - Sending an extended query prepared with \bind using a ';' on a
> newline, though this is not specific to pipelining. The scanned
> semicolon triggers the call to SendQuery, processing the buffered
> extended query. It's a bit unusual but that's the current behaviour.
The tests could be much more organized, particularly for the "sinple"
and "multiple" and COPY cases, rather than being treated as two
different groups at different locations of psql_pipeline.sql. I've
spent some time reorganizing all that.
A second thing that was a bit itchy is the use of ";" for what's a
semicolon, and we use this term in the psql docs to refer to queries
terminated by that. The whole paragraph could be simplified a bit
more, mentioning that everything in a pipeline uses the extended
protocol, while \bind & co are more like options. The description of
PIPELINE_COMMAND_COUNT could be simpler, and the part about the
pending results can be more general now so I've removed it.
With all that set, I've applied the patch. If you have more
suggestions, please feel free to mention them.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-19 13:05 Daniel Verite <[email protected]>
parent: Michael Paquier <[email protected]>
0 siblings, 0 replies; 55+ messages in thread
From: Daniel Verite @ 2025-03-19 13:05 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; Anthonin Bonnefoy <[email protected]>; pgsql-hackers
Michael Paquier wrote:
> Perhaps an \extended command that behaves outside a pipeline makes
> sense to force the use of queries without parameters to use the
> extended mode, but I cannot get much excited about the concept knowing
> all the meta-commands we have now (not talking about the pipeline
> part, which is different, as we can treat queries in batches).
When psql started supporting the extended query protocol, the question
of enabling it more globally rather than query-by-query was discussed
a bit [1]. The idea was to switch to it with a setting or a variable
rather than a metacommand.
Some pros and cons were mentioned in the thread, but on the whole
it was not convincing enough to get implemented.
[1]
https://www.postgresql.org/message-id/e8dd1cd5-0e04-3598-0518-a605159fe314%40enterprisedb.com
Best regards,
--
Daniel Vérité
https://postgresql.verite.pro/
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2025-04-11 20:32 Noah Misch <[email protected]>
parent: Alexander Korotkov <[email protected]>
1 sibling, 1 reply; 55+ messages in thread
From: Noah Misch @ 2025-04-11 20:32 UTC (permalink / raw)
To: Alexander Korotkov <[email protected]>; +Cc: Pavel Borisov <[email protected]>; Dagfinn Ilmari Mannsåker <[email protected]>; Andrei Lepikhov <[email protected]>; jian he <[email protected]>; Artur Zakirov <[email protected]>; Alexander Lakhin <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
On Tue, Oct 22, 2024 at 08:33:24PM +0300, Alexander Korotkov wrote:
> On Tue, Oct 22, 2024 at 6:10 PM Pavel Borisov <[email protected]> wrote:
> > On Tue, 22 Oct 2024 at 11:34, Alexander Korotkov <[email protected]> wrote:
> >> I'm going to push this if no objections.
(This became commit b85a9d0.)
> + /* Call delete_rel_type_cache() if we actually cleared something */
> + if (hadTupDescOrOpclass)
> + delete_rel_type_cache_if_needed(typentry);
I think the intent was to maintain the invariant that a RelIdToTypeIdCacheHash
entry exists if and only if certain kinds of data appear in the TypeCacheHash
entry. However, TypeCacheOpcCallback() clears TCFLAGS_OPERATOR_FLAGS without
maintaining RelIdToTypeIdCacheHash. Is it right to do that?
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2025-04-11 21:43 Alexander Korotkov <[email protected]>
parent: Noah Misch <[email protected]>
0 siblings, 1 reply; 55+ messages in thread
From: Alexander Korotkov @ 2025-04-11 21:43 UTC (permalink / raw)
To: Noah Misch <[email protected]>; +Cc: Pavel Borisov <[email protected]>; Dagfinn Ilmari Mannsåker <[email protected]>; Andrei Lepikhov <[email protected]>; jian he <[email protected]>; Artur Zakirov <[email protected]>; Alexander Lakhin <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
On Fri, Apr 11, 2025 at 11:32 PM Noah Misch <[email protected]> wrote:
>
> On Tue, Oct 22, 2024 at 08:33:24PM +0300, Alexander Korotkov wrote:
> > On Tue, Oct 22, 2024 at 6:10 PM Pavel Borisov <[email protected]> wrote:
> > > On Tue, 22 Oct 2024 at 11:34, Alexander Korotkov <[email protected]> wrote:
> > >> I'm going to push this if no objections.
>
> (This became commit b85a9d0.)
>
> > + /* Call delete_rel_type_cache() if we actually cleared something */
> > + if (hadTupDescOrOpclass)
> > + delete_rel_type_cache_if_needed(typentry);
>
> I think the intent was to maintain the invariant that a RelIdToTypeIdCacheHash
> entry exists if and only if certain kinds of data appear in the TypeCacheHash
> entry. However, TypeCacheOpcCallback() clears TCFLAGS_OPERATOR_FLAGS without
> maintaining RelIdToTypeIdCacheHash. Is it right to do that?
Thank you for the question. I'll recheck this in next couple of days.
------
Regards,
Alexander Korotkov
Supabase
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-04-16 14:31 a.kozhemyakin <[email protected]>
0 siblings, 1 reply; 55+ messages in thread
From: a.kozhemyakin @ 2025-04-16 14:31 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; Daniel Verite <[email protected]>; +Cc: Anthonin Bonnefoy <[email protected]>; Jelte Fennema-Nio <[email protected]>; pgsql-hackers
Hello,
After commit 2cce0fe on master
When executing query:
psql postgres <<EOF
CREATE TABLE psql_pipeline();
\startpipeline
COPY psql_pipeline FROM STDIN;
SELECT 'val1';
\syncpipeline
\getresults
EOF
ERROR: unexpected message type 0x50 during COPY from stdin
CONTEXT: COPY psql_pipeline, line 1
Pipeline aborted, command did not run
psql: common.c:1510: discardAbortedPipelineResults: Assertion `res ==
((void *)0) || result_status == PGRES_PIPELINE_ABORTED' failed.
Aborted (core dumped)
The psql crashes with the stack trace:
(gdb) bt
#0 __pthread_kill_implementation (no_tid=0, signo=6,
threadid=<optimized out>) at ./nptl/pthread_kill.c:44
#1 __pthread_kill_internal (signo=6, threadid=<optimized out>) at
./nptl/pthread_kill.c:78
#2 __GI___pthread_kill (threadid=<optimized out>, signo=signo@entry=6)
at ./nptl/pthread_kill.c:89
#3 0x0000760edd24527e in __GI_raise (sig=sig@entry=6) at
../sysdeps/posix/raise.c:26
#4 0x0000760edd2288ff in __GI_abort () at ./stdlib/abort.c:79
#5 0x0000760edd22881b in __assert_fail_base (fmt=0x760edd3d01e8
"%s%s%s:%u: %s%sAssertion `%s' failed.\n%n",
assertion=assertion@entry=0x5ba46ab79850 "res == ((void *)0) ||
result_status == PGRES_PIPELINE_ABORTED", file=file@entry=0x5ba46ab6fcad
"common.c",
line=line@entry=1510, function=function@entry=0x5ba46ab9c780
<__PRETTY_FUNCTION__.3> "discardAbortedPipelineResults") at
./assert/assert.c:96
#6 0x0000760edd23b517 in __assert_fail
(assertion=assertion@entry=0x5ba46ab79850 "res == ((void *)0) ||
result_status == PGRES_PIPELINE_ABORTED",
file=file@entry=0x5ba46ab6fcad "common.c", line=line@entry=1510,
function=function@entry=0x5ba46ab9c780 <__PRETTY_FUNCTION__.3>
"discardAbortedPipelineResults") at ./assert/assert.c:105
#7 0x00005ba46ab2bd40 in discardAbortedPipelineResults () at common.c:1510
#8 ExecQueryAndProcessResults (query=query@entry=0x5ba4a2ec1e10 "SELECT
'val1';", elapsed_msec=elapsed_msec@entry=0x7ffeb07262a8,
svpt_gone_p=svpt_gone_p@entry=0x7ffeb07262a7,
is_watch=is_watch@entry=false, min_rows=min_rows@entry=0,
opt=opt@entry=0x0, printQueryFout=0x0)
at common.c:1811
#9 0x00005ba46ab2983f in SendQuery (query=0x5ba4a2ec1e10 "SELECT
'val1';") at common.c:1212
#10 0x00005ba46ab3f66a in MainLoop (source=source@entry=0x760edd4038e0
<_IO_2_1_stdin_>) at mainloop.c:515
#11 0x00005ba46ab23f2a in process_file (filename=0x0,
use_relative_path=use_relative_path@entry=false) at command.c:4870
#12 0x00005ba46ab1e9d9 in main (argc=<optimized out>,
argv=0x7ffeb07269d8) at startup.c:420
06.03.2025 11:20, Michael Paquier пишет:
> On Wed, Mar 05, 2025 at 03:25:12PM +0100, Daniel Verite wrote:
>> Anthonin Bonnefoy wrote:
>>> I do see the idea to make it easier to convert existing scripts into
>>> using pipelining. The main focus of the initial implementation was
>>> more on protocol regression tests with psql, so that's not necessarily
>>> something I had in mind.
>> Understood. Yet pipelining can accelerate considerably certain scripts
>> when client-server latency is an issue. We should expect end users to
>> benefit from it too.
> That was not a test case we had in mind originally here, but if it is
> possible to keep the implementation simple while supporting your
> demand, well, let's do it. If it's not that straight-forward, let's
> use the new meta-command, forbidding \g and \gx based on your
> arguments from upthread.
> --
> Michael
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-04-16 16:18 Michael Paquier <[email protected]>
parent: a.kozhemyakin <[email protected]>
0 siblings, 1 reply; 55+ messages in thread
From: Michael Paquier @ 2025-04-16 16:18 UTC (permalink / raw)
To: a.kozhemyakin <[email protected]>; +Cc: Daniel Verite <[email protected]>; Anthonin Bonnefoy <[email protected]>; Jelte Fennema-Nio <[email protected]>; pgsql-hackers
On Wed, Apr 16, 2025 at 09:31:59PM +0700, a.kozhemyakin wrote:
> After commit 2cce0fe on master
>
> ERROR: unexpected message type 0x50 during COPY from stdin
> CONTEXT: COPY psql_pipeline, line 1
> Pipeline aborted, command did not run
> psql: common.c:1510: discardAbortedPipelineResults: Assertion `res == ((void
> *)0) || result_status == PGRES_PIPELINE_ABORTED' failed.
> Aborted (core dumped)
Reproduced here, thanks for the report. I'll look at that at the
beginning of next week, adding an open item for now.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2025-04-21 01:54 Alexander Korotkov <[email protected]>
parent: Alexander Korotkov <[email protected]>
0 siblings, 1 reply; 55+ messages in thread
From: Alexander Korotkov @ 2025-04-21 01:54 UTC (permalink / raw)
To: Noah Misch <[email protected]>; +Cc: Pavel Borisov <[email protected]>; Dagfinn Ilmari Mannsåker <[email protected]>; Andrei Lepikhov <[email protected]>; jian he <[email protected]>; Artur Zakirov <[email protected]>; Alexander Lakhin <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
Hi, Noah!
On Sat, Apr 12, 2025 at 12:43 AM Alexander Korotkov
<[email protected]> wrote:
>
> On Fri, Apr 11, 2025 at 11:32 PM Noah Misch <[email protected]> wrote:
> >
> > On Tue, Oct 22, 2024 at 08:33:24PM +0300, Alexander Korotkov wrote:
> > > On Tue, Oct 22, 2024 at 6:10 PM Pavel Borisov <[email protected]> wrote:
> > > > On Tue, 22 Oct 2024 at 11:34, Alexander Korotkov <[email protected]> wrote:
> > > >> I'm going to push this if no objections.
> >
> > (This became commit b85a9d0.)
> >
> > > + /* Call delete_rel_type_cache() if we actually cleared something */
> > > + if (hadTupDescOrOpclass)
> > > + delete_rel_type_cache_if_needed(typentry);
> >
> > I think the intent was to maintain the invariant that a RelIdToTypeIdCacheHash
> > entry exists if and only if certain kinds of data appear in the TypeCacheHash
> > entry. However, TypeCacheOpcCallback() clears TCFLAGS_OPERATOR_FLAGS without
> > maintaining RelIdToTypeIdCacheHash. Is it right to do that?
>
> Thank you for the question. I'll recheck this in next couple of days.
Sorry for the delay. Generally, your finding is correct. But, I
didn't manage to reproduce the situation, where existing code leads to
real error. In order to have it, we must have typcache entry without
TCFLAGS_HAVE_PG_TYPE_DATA and tupDesc, but with some of
TCFLAGS_OPERATOR_FLAGS. Reseting TCFLAGS_HAVE_PG_TYPE_DATA for a
composite type doesn't seem to be possible without resetting the rest
at the same time.
Nevertheless, I think it would be fragile to leave the current code
"as is". If even there is no case of real error (or it's just me
didn't manage to find it), it could appear after further changes of
type cache code. So, the fix is attached.
------
Regards,
Alexander Korotkov
Supabase
Attachments:
[application/octet-stream] v1-0001-Maintain-RelIdToTypeIdCacheHash-in-TypeCacheOpcCa.patch (2.5K, ../../CAPpHfdv5x5VpiBRuAGYUhyT417chXN2G9Dt+=6m+Odi8A5_R=Q@mail.gmail.com/2-v1-0001-Maintain-RelIdToTypeIdCacheHash-in-TypeCacheOpcCa.patch)
download | inline diff:
From f44dac0d623783aa7bb3ab03eb9c91bb76d1ae87 Mon Sep 17 00:00:00 2001
From: Alexander Korotkov <[email protected]>
Date: Mon, 21 Apr 2025 01:40:32 +0300
Subject: [PATCH v1] Maintain RelIdToTypeIdCacheHash in TypeCacheOpcCallback()
b85a9d046efd introduced a new RelIdToTypeIdCacheHash, whose entries should
exist for typecache entries with TCFLAGS_HAVE_PG_TYPE_DATA flag set or any
of TCFLAGS_OPERATOR_FLAGS set or tupDesc set. However, TypeCacheOpcCallback(),
which resets TCFLAGS_OPERATOR_FLAGS, was forgotten to update
RelIdToTypeIdCacheHash.
This commit adds a delete_rel_type_cache_if_needed() call to the
TypeCacheOpcCallback() function to maintain RelIdToTypeIdCacheHash after
resetting TCFLAGS_OPERATOR_FLAGS.
Also, this commit fixes the name of the delete_rel_type_cache_if_needed()
function in its mentions in the comments.
Reported-by: Noah Misch
Discussion: https://postgr.es/m/20250411203241.e9.nmisch%40google.com
---
src/backend/utils/cache/typcache.c | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
diff --git a/src/backend/utils/cache/typcache.c b/src/backend/utils/cache/typcache.c
index ae65a1cce06..560f5595fda 100644
--- a/src/backend/utils/cache/typcache.c
+++ b/src/backend/utils/cache/typcache.c
@@ -2395,7 +2395,7 @@ InvalidateCompositeTypeCacheEntry(TypeCacheEntry *typentry)
/* Reset equality/comparison/hashing validity information */
typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
- /* Call delete_rel_type_cache() if we actually cleared something */
+ /* Call delete_rel_type_cache_if_needed() if we actually cleared something */
if (hadTupDescOrOpclass)
delete_rel_type_cache_if_needed(typentry);
}
@@ -2542,7 +2542,7 @@ TypeCacheTypCallback(Datum arg, int cacheid, uint32 hashvalue)
TCFLAGS_CHECKED_DOMAIN_CONSTRAINTS);
/*
- * Call delete_rel_type_cache() if we cleaned
+ * Call delete_rel_type_cache_if_needed() if we cleaned
* TCFLAGS_HAVE_PG_TYPE_DATA flag previously.
*/
if (hadPgTypeData)
@@ -2576,8 +2576,17 @@ TypeCacheOpcCallback(Datum arg, int cacheid, uint32 hashvalue)
hash_seq_init(&status, TypeCacheHash);
while ((typentry = (TypeCacheEntry *) hash_seq_search(&status)) != NULL)
{
+ bool hadOpclass = (typentry->flags & TCFLAGS_OPERATOR_FLAGS);
+
/* Reset equality/comparison/hashing validity information */
typentry->flags &= ~TCFLAGS_OPERATOR_FLAGS;
+
+ /*
+ * Call delete_rel_type_cache_if_needed() if we actually cleared
+ * something
+ */
+ if (hadOpclass)
+ delete_rel_type_cache_if_needed(typentry);
}
}
--
2.39.5 (Apple Git-154)
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-04-22 12:37 Anthonin Bonnefoy <[email protected]>
parent: Michael Paquier <[email protected]>
0 siblings, 0 replies; 55+ messages in thread
From: Anthonin Bonnefoy @ 2025-04-22 12:37 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: a.kozhemyakin <[email protected]>; Daniel Verite <[email protected]>; Jelte Fennema-Nio <[email protected]>; pgsql-hackers
On Tue, Apr 22, 2025 at 2:06 AM Michael Paquier <[email protected]> wrote:
> I am wondering if we could not be smarter with the handling of
> the counters, but I really doubt that there is much more we can do
> under a PGRES_FATAL_ERROR.
One thing that bothers me is that the reported error is silently
discarded within discardAbortedPipelineResults.
psql -f bug_assertion.sql
psql:bug_assertion.sql:7: ERROR: unexpected message type 0x50 during
COPY from stdin
CONTEXT: COPY psql_pipeline, line 1
psql:bug_assertion.sql:7: Pipeline aborted, command did not run
This should ideally report the "FATAL: terminating connection because
protocol synchronization was lost" sent by the backend process.
Also, we still have a triggered assertion failure with the following:
CREATE TABLE psql_pipeline(a text);
\startpipeline
COPY psql_pipeline FROM STDIN;
SELECT 'val1';
\syncpipeline
\endpipeline
...
Assertion failed: (pset.piped_syncs == 0), function
ExecQueryAndProcessResults, file common.c, line 2153.
A possible alternative could be to abort discardAbortedPipelineResults
when we encounter a res != NULL + FATAL error and let the outer loop
handle it. As you said, the pipeline flow is borked so there's not
much to salvage. The outer loop would read and print all error
messages until the closed connection is detected. Then,
CheckConnection will reset the connection which will reset the
pipeline state.
While testing this change, I was initially looking for the "FATAL:
terminating connection because protocol synchronization was lost"
message in the tests. However, this was failing on Windows[1] as the
FATAL message wasn't reported on stderr. I'm not sure why yet.
[1]: https://cirrus-ci.com/task/5051031505076224?logs=check_world#L240-L246
Attachments:
[application/octet-stream] v02-0001-PATCH-psql-Fix-assertion-failure-with-pipeline-m.patch (3.7K, ../../CAO6_XqoQaZVHEw1dFgvOxwectqsNo4QE1tQ6rLD-YUzFpT_+3g@mail.gmail.com/2-v02-0001-PATCH-psql-Fix-assertion-failure-with-pipeline-m.patch)
download | inline diff:
From 5d37f2616c82c6525d656149d383ef01a6d7518c Mon Sep 17 00:00:00 2001
From: Michael Paquier <[email protected]>
Date: Mon, 21 Apr 2025 15:17:43 +0900
Subject: [PATCH] psql: Fix assertion failure with pipeline mode
---
src/bin/psql/common.c | 17 ++++++++++++
src/bin/psql/t/001_basic.pl | 55 ++++++++++++++++++++++++++++++++++---
2 files changed, 68 insertions(+), 4 deletions(-)
diff --git a/src/bin/psql/common.c b/src/bin/psql/common.c
index 21d660a8961..0aab02ee32e 100644
--- a/src/bin/psql/common.c
+++ b/src/bin/psql/common.c
@@ -1478,6 +1478,23 @@ discardAbortedPipelineResults(void)
*/
return res;
}
+ else if (res != NULL && result_status == PGRES_FATAL_ERROR)
+ {
+ /*
+ * We have a fatal error sent by the backend and we can't recover
+ * from this state. Instead, return the last fatal error and let
+ * the outer loop handle it.
+ */
+ PGresult *fatal_res PG_USED_FOR_ASSERTS_ONLY;
+
+ /*
+ * Fetch result to consume the end of the current query being
+ * processed.
+ */
+ fatal_res = PQgetResult(pset.db);
+ Assert(fatal_res == NULL);
+ return res;
+ }
else if (res == NULL)
{
/* A query was processed, decrement the counters */
diff --git a/src/bin/psql/t/001_basic.pl b/src/bin/psql/t/001_basic.pl
index 739cb439708..8d258c00c5e 100644
--- a/src/bin/psql/t/001_basic.pl
+++ b/src/bin/psql/t/001_basic.pl
@@ -34,11 +34,13 @@ sub psql_fails_like
{
local $Test::Builder::Level = $Test::Builder::Level + 1;
- my ($node, $sql, $expected_stderr, $test_name) = @_;
+ my ($node, $sql, $expected_stderr, $test_name, $replication) = @_;
+
+ # Use the context of a WAL sender, if requested by the caller.
+ $replication = '' unless defined($replication);
- # Use the context of a WAL sender, some of the tests rely on that.
my ($ret, $stdout, $stderr) =
- $node->psql('postgres', $sql, replication => 'database');
+ $node->psql('postgres', $sql, replication => $replication);
isnt($ret, 0, "$test_name: exit code not 0");
like($stderr, $expected_stderr, "$test_name: matches");
@@ -79,7 +81,8 @@ psql_fails_like(
$node,
'START_REPLICATION 0/0',
qr/unexpected PQresultStatus: 8$/,
- 'handling of unexpected PQresultStatus');
+ 'handling of unexpected PQresultStatus',
+ 'database');
# test \timing
psql_like(
@@ -481,4 +484,48 @@ psql_like($node, "copy (values ('foo'),('bar')) to stdout \\g | $pipe_cmd",
my $c4 = slurp_file($g_file);
like($c4, qr/foo.*bar/s);
+# Tests with pipelines. These trigger FATAL failures in the backend,
+# so they cannot be tested through the SQL regression tests.
+$node->safe_psql('postgres', 'CREATE TABLE psql_pipeline()');
+psql_fails_like(
+ $node,
+ qq{\\startpipeline
+COPY psql_pipeline FROM STDIN;
+SELECT 'val1';
+\\syncpipeline
+\\getresults
+\\endpipeline},
+ qr/server closed the connection unexpectedly/,
+ 'handling of protocol synchronization loss with pipelines');
+psql_fails_like(
+ $node,
+ qq{\\startpipeline
+COPY psql_pipeline FROM STDIN \\bind \\sendpipeline
+SELECT 'val1' \\bind \\sendpipeline
+\\syncpipeline
+\\getresults
+\\endpipeline},
+ qr/server closed the connection unexpectedly/,
+ 'handling of protocol synchronization loss with pipelines');
+
+# This time, test without the \getresults
+psql_fails_like(
+ $node,
+ qq{\\startpipeline
+COPY psql_pipeline FROM STDIN;
+SELECT 'val1';
+\\syncpipeline
+\\endpipeline},
+ qr/server closed the connection unexpectedly/,
+ 'handling of protocol synchronization loss with pipelines');
+psql_fails_like(
+ $node,
+ qq{\\startpipeline
+COPY psql_pipeline FROM STDIN \\bind \\sendpipeline
+SELECT 'val1' \\bind \\sendpipeline
+\\syncpipeline
+\\endpipeline},
+ qr/server closed the connection unexpectedly/,
+ 'handling of protocol synchronization loss with pipelines');
+
done_testing();
--
2.39.5 (Apple Git-154)
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2025-04-29 00:56 Noah Misch <[email protected]>
parent: Alexander Korotkov <[email protected]>
0 siblings, 1 reply; 55+ messages in thread
From: Noah Misch @ 2025-04-29 00:56 UTC (permalink / raw)
To: Alexander Korotkov <[email protected]>; +Cc: Pavel Borisov <[email protected]>; Dagfinn Ilmari Mannsåker <[email protected]>; Andrei Lepikhov <[email protected]>; jian he <[email protected]>; Artur Zakirov <[email protected]>; Alexander Lakhin <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
On Mon, Apr 21, 2025 at 04:54:08AM +0300, Alexander Korotkov wrote:
> On Sat, Apr 12, 2025 at 12:43 AM Alexander Korotkov <[email protected]> wrote:
> > On Fri, Apr 11, 2025 at 11:32 PM Noah Misch <[email protected]> wrote:
> > > On Tue, Oct 22, 2024 at 08:33:24PM +0300, Alexander Korotkov wrote:
> > > > On Tue, Oct 22, 2024 at 6:10 PM Pavel Borisov <[email protected]> wrote:
> > > > > On Tue, 22 Oct 2024 at 11:34, Alexander Korotkov <[email protected]> wrote:
> > > > >> I'm going to push this if no objections.
> > >
> > > (This became commit b85a9d0.)
> > >
> > > > + /* Call delete_rel_type_cache() if we actually cleared something */
> > > > + if (hadTupDescOrOpclass)
> > > > + delete_rel_type_cache_if_needed(typentry);
> > >
> > > I think the intent was to maintain the invariant that a RelIdToTypeIdCacheHash
> > > entry exists if and only if certain kinds of data appear in the TypeCacheHash
> > > entry. However, TypeCacheOpcCallback() clears TCFLAGS_OPERATOR_FLAGS without
> > > maintaining RelIdToTypeIdCacheHash. Is it right to do that?
> Sorry for the delay. Generally, your finding is correct. But, I
> didn't manage to reproduce the situation, where existing code leads to
> real error. In order to have it, we must have typcache entry without
> TCFLAGS_HAVE_PG_TYPE_DATA and tupDesc, but with some of
> TCFLAGS_OPERATOR_FLAGS.
That makes sense.
> Reseting TCFLAGS_HAVE_PG_TYPE_DATA for a
> composite type doesn't seem to be possible without resetting the rest
> at the same time.
>
> Nevertheless, I think it would be fragile to leave the current code
> "as is". If even there is no case of real error (or it's just me
> didn't manage to find it), it could appear after further changes of
> type cache code. So, the fix is attached.
This change looks appropriate. Thanks.
^ permalink raw reply [nested|flat] 55+ messages in thread
* Re: type cache cleanup improvements
@ 2025-04-29 10:00 Alexander Korotkov <[email protected]>
parent: Noah Misch <[email protected]>
0 siblings, 0 replies; 55+ messages in thread
From: Alexander Korotkov @ 2025-04-29 10:00 UTC (permalink / raw)
To: Noah Misch <[email protected]>; +Cc: Pavel Borisov <[email protected]>; Dagfinn Ilmari Mannsåker <[email protected]>; Andrei Lepikhov <[email protected]>; jian he <[email protected]>; Artur Zakirov <[email protected]>; Alexander Lakhin <[email protected]>; Teodor Sigaev <[email protected]>; pgsql-hackers; Aleksander Alekseev <[email protected]>; Tom Lane <[email protected]>; Michael Paquier <[email protected]>
Hi, Noah!
On Tue, Apr 29, 2025 at 3:56 AM Noah Misch <[email protected]> wrote:
> On Mon, Apr 21, 2025 at 04:54:08AM +0300, Alexander Korotkov wrote:
> > On Sat, Apr 12, 2025 at 12:43 AM Alexander Korotkov <[email protected]> wrote:
> > > On Fri, Apr 11, 2025 at 11:32 PM Noah Misch <[email protected]> wrote:
> > > > On Tue, Oct 22, 2024 at 08:33:24PM +0300, Alexander Korotkov wrote:
> > > > > On Tue, Oct 22, 2024 at 6:10 PM Pavel Borisov <[email protected]> wrote:
> > > > > > On Tue, 22 Oct 2024 at 11:34, Alexander Korotkov <[email protected]> wrote:
> > > > > >> I'm going to push this if no objections.
> > > >
> > > > (This became commit b85a9d0.)
> > > >
> > > > > + /* Call delete_rel_type_cache() if we actually cleared something */
> > > > > + if (hadTupDescOrOpclass)
> > > > > + delete_rel_type_cache_if_needed(typentry);
> > > >
> > > > I think the intent was to maintain the invariant that a RelIdToTypeIdCacheHash
> > > > entry exists if and only if certain kinds of data appear in the TypeCacheHash
> > > > entry. However, TypeCacheOpcCallback() clears TCFLAGS_OPERATOR_FLAGS without
> > > > maintaining RelIdToTypeIdCacheHash. Is it right to do that?
>
> > Sorry for the delay. Generally, your finding is correct. But, I
> > didn't manage to reproduce the situation, where existing code leads to
> > real error. In order to have it, we must have typcache entry without
> > TCFLAGS_HAVE_PG_TYPE_DATA and tupDesc, but with some of
> > TCFLAGS_OPERATOR_FLAGS.
>
> That makes sense.
>
> > Reseting TCFLAGS_HAVE_PG_TYPE_DATA for a
> > composite type doesn't seem to be possible without resetting the rest
> > at the same time.
> >
> > Nevertheless, I think it would be fragile to leave the current code
> > "as is". If even there is no case of real error (or it's just me
> > didn't manage to find it), it could appear after further changes of
> > type cache code. So, the fix is attached.
>
> This change looks appropriate. Thanks.
Thank you for your feedback!
------
Regards,
Alexander Korotkov
Supabase
^ permalink raw reply [nested|flat] 55+ messages in thread
* [PATCH v51 07/10] Check for transaction block early in ExecRepack
@ 2026-04-03 15:23 Srinath Reddy Sadipiralla <[email protected]>
0 siblings, 0 replies; 55+ messages in thread
From: Srinath Reddy Sadipiralla @ 2026-04-03 15:23 UTC (permalink / raw)
Currently, executing REPACK (CONCURRENTLY) without a table name inside a
transaction block throws the error "REPACK CONCURRENTLY requires explicit
table name" instead of the expected transaction block error. This occurs
because ExecRepack() validates the parsed options and missing relation
before verifying the transaction state.
This behavior is inconsistent with other utility commands like VACUUM
,REINDEX, etc; which invoke PreventInTransactionBlock() at the very start
of their execution to properly reject execution inside user transactions
before validating targets.
Add PreventInTransactionBlock to the top of ExecRepack() to enforce the
transaction block restriction early. This prevents the user from fixing
a missing table error only to immediately hit a transaction block error,
and also ensures consistency with rest of the commands.
Author: Srinath Reddy Sadipiralla <[email protected]>
---
src/backend/commands/repack.c | 26 ++++++++++++--------------
1 file changed, 12 insertions(+), 14 deletions(-)
diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c
index d6e446d582d..d4c1f0e7652 100644
--- a/src/backend/commands/repack.c
+++ b/src/backend/commands/repack.c
@@ -292,6 +292,18 @@ ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel)
MyProc->statusFlags |= PROC_IN_CONCURRENT_REPACK;
ProcGlobal->statusFlags[MyProc->pgxactoff] = MyProc->statusFlags;
LWLockRelease(ProcArrayLock);
+
+ /*
+ * Make sure we're not in a transaction block.
+ *
+ * The reason is that repack_setup_logical_decoding() could wait
+ * indefinitely for our XID to complete. (The deadlock detector would
+ * not recognize it because we'd be waiting for ourselves, i.e. no
+ * real lock conflict.) It would be possible to run in a transaction
+ * block if we had no XID, but this restriction is simpler for users
+ * to understand and we don't lose anything.
+ */
+ PreventInTransactionBlock(isTopLevel, "REPACK (CONCURRENTLY)");
}
/*
@@ -523,21 +535,7 @@ cluster_rel(RepackCommand cmd, Relation OldHeap, Oid indexOid,
* replica index OID.
*/
if (concurrent)
- {
- /*
- * Make sure we're not in a transaction block.
- *
- * The reason is that repack_setup_logical_decoding() could wait
- * indefinitely for our XID to complete. (The deadlock detector would
- * not recognize it because we'd be waiting for ourselves, i.e. no
- * real lock conflict.) It would be possible to run in a transaction
- * block if we had no XID, but this restriction is simpler for users
- * to understand and we don't lose anything.
- */
- PreventInTransactionBlock(isTopLevel, "REPACK (CONCURRENTLY)");
-
check_repack_concurrently_requirements(OldHeap, &ident_idx);
- }
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
--
2.47.3
--r2slln3zpmilwu22
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
filename="v51-0008-Introduce-an-option-to-make-logical-replication-.patch"
^ permalink raw reply [nested|flat] 55+ messages in thread
* [PATCH v51 07/10] Check for transaction block early in ExecRepack
@ 2026-04-03 15:23 Srinath Reddy Sadipiralla <[email protected]>
0 siblings, 0 replies; 55+ messages in thread
From: Srinath Reddy Sadipiralla @ 2026-04-03 15:23 UTC (permalink / raw)
Currently, executing REPACK (CONCURRENTLY) without a table name inside a
transaction block throws the error "REPACK CONCURRENTLY requires explicit
table name" instead of the expected transaction block error. This occurs
because ExecRepack() validates the parsed options and missing relation
before verifying the transaction state.
This behavior is inconsistent with other utility commands like VACUUM
,REINDEX, etc; which invoke PreventInTransactionBlock() at the very start
of their execution to properly reject execution inside user transactions
before validating targets.
Add PreventInTransactionBlock to the top of ExecRepack() to enforce the
transaction block restriction early. This prevents the user from fixing
a missing table error only to immediately hit a transaction block error,
and also ensures consistency with rest of the commands.
Author: Srinath Reddy Sadipiralla <[email protected]>
---
src/backend/commands/repack.c | 26 ++++++++++++--------------
1 file changed, 12 insertions(+), 14 deletions(-)
diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c
index d6e446d582d..d4c1f0e7652 100644
--- a/src/backend/commands/repack.c
+++ b/src/backend/commands/repack.c
@@ -292,6 +292,18 @@ ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel)
MyProc->statusFlags |= PROC_IN_CONCURRENT_REPACK;
ProcGlobal->statusFlags[MyProc->pgxactoff] = MyProc->statusFlags;
LWLockRelease(ProcArrayLock);
+
+ /*
+ * Make sure we're not in a transaction block.
+ *
+ * The reason is that repack_setup_logical_decoding() could wait
+ * indefinitely for our XID to complete. (The deadlock detector would
+ * not recognize it because we'd be waiting for ourselves, i.e. no
+ * real lock conflict.) It would be possible to run in a transaction
+ * block if we had no XID, but this restriction is simpler for users
+ * to understand and we don't lose anything.
+ */
+ PreventInTransactionBlock(isTopLevel, "REPACK (CONCURRENTLY)");
}
/*
@@ -523,21 +535,7 @@ cluster_rel(RepackCommand cmd, Relation OldHeap, Oid indexOid,
* replica index OID.
*/
if (concurrent)
- {
- /*
- * Make sure we're not in a transaction block.
- *
- * The reason is that repack_setup_logical_decoding() could wait
- * indefinitely for our XID to complete. (The deadlock detector would
- * not recognize it because we'd be waiting for ourselves, i.e. no
- * real lock conflict.) It would be possible to run in a transaction
- * block if we had no XID, but this restriction is simpler for users
- * to understand and we don't lose anything.
- */
- PreventInTransactionBlock(isTopLevel, "REPACK (CONCURRENTLY)");
-
check_repack_concurrently_requirements(OldHeap, &ident_idx);
- }
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
--
2.47.3
--r2slln3zpmilwu22
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
filename="v51-0008-Introduce-an-option-to-make-logical-replication-.patch"
^ permalink raw reply [nested|flat] 55+ messages in thread
* [PATCH v53 5/7] Check for transaction block early in ExecRepack
@ 2026-04-03 15:23 Srinath Reddy Sadipiralla <[email protected]>
0 siblings, 0 replies; 55+ messages in thread
From: Srinath Reddy Sadipiralla @ 2026-04-03 15:23 UTC (permalink / raw)
Currently, executing REPACK (CONCURRENTLY) without a table name inside a
transaction block throws the error "REPACK CONCURRENTLY requires explicit
table name" instead of the expected transaction block error. This occurs
because ExecRepack() validates the parsed options and missing relation
before verifying the transaction state.
This behavior is inconsistent with other utility commands like VACUUM
,REINDEX, etc; which invoke PreventInTransactionBlock() at the very start
of their execution to properly reject execution inside user transactions
before validating targets.
Add PreventInTransactionBlock to the top of ExecRepack() to enforce the
transaction block restriction early. This prevents the user from fixing
a missing table error only to immediately hit a transaction block error,
and also ensures consistency with rest of the commands.
Author: Srinath Reddy Sadipiralla <[email protected]>
---
src/backend/commands/repack.c | 27 +++++++++++++++------------
1 file changed, 15 insertions(+), 12 deletions(-)
diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c
index f2759cdbef1..8d9b2b2e370 100644
--- a/src/backend/commands/repack.c
+++ b/src/backend/commands/repack.c
@@ -279,6 +279,21 @@ ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel)
/* Determine the lock mode to use. */
lockmode = RepackLockLevel((params.options & CLUOPT_CONCURRENT) != 0);
+ if ((params.options & CLUOPT_CONCURRENT) != 0)
+ {
+ /*
+ * Make sure we're not in a transaction block.
+ *
+ * The reason is that repack_setup_logical_decoding() could wait
+ * indefinitely for our XID to complete. (The deadlock detector would
+ * not recognize it because we'd be waiting for ourselves, i.e. no
+ * real lock conflict.) It would be possible to run in a transaction
+ * block if we had no XID, but this restriction is simpler for users
+ * to understand and we don't lose any functionality.
+ */
+ PreventInTransactionBlock(isTopLevel, "REPACK (CONCURRENTLY)");
+ }
+
/*
* If a single relation is specified, process it and we're done ... unless
* the relation is a partitioned table, in which case we fall through.
@@ -511,19 +526,7 @@ cluster_rel(RepackCommand cmd, Relation OldHeap, Oid indexOid,
* replica index OID.
*/
if (concurrent)
- {
- /*
- * Make sure we're not in a transaction block.
- *
- * The reason is that repack_setup_logical_decoding() could deadlock
- * if there's an XID already assigned. It would be possible to run in
- * a transaction block if we had no XID, but this restriction is
- * simpler for users to understand and we don't lose anything.
- */
- PreventInTransactionBlock(isTopLevel, "REPACK (CONCURRENTLY)");
-
check_repack_concurrently_requirements(OldHeap, &ident_idx);
- }
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
--
2.47.3
--qr3jlalmmcpkiodg
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
filename="v53-0006-Error-out-any-process-that-would-block-at-REPACK.patch"
^ permalink raw reply [nested|flat] 55+ messages in thread
* [PATCH v53 5/7] Check for transaction block early in ExecRepack
@ 2026-04-03 15:23 Srinath Reddy Sadipiralla <[email protected]>
0 siblings, 0 replies; 55+ messages in thread
From: Srinath Reddy Sadipiralla @ 2026-04-03 15:23 UTC (permalink / raw)
Currently, executing REPACK (CONCURRENTLY) without a table name inside a
transaction block throws the error "REPACK CONCURRENTLY requires explicit
table name" instead of the expected transaction block error. This occurs
because ExecRepack() validates the parsed options and missing relation
before verifying the transaction state.
This behavior is inconsistent with other utility commands like VACUUM
,REINDEX, etc; which invoke PreventInTransactionBlock() at the very start
of their execution to properly reject execution inside user transactions
before validating targets.
Add PreventInTransactionBlock to the top of ExecRepack() to enforce the
transaction block restriction early. This prevents the user from fixing
a missing table error only to immediately hit a transaction block error,
and also ensures consistency with rest of the commands.
Author: Srinath Reddy Sadipiralla <[email protected]>
---
src/backend/commands/repack.c | 27 +++++++++++++++------------
1 file changed, 15 insertions(+), 12 deletions(-)
diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c
index f2759cdbef1..8d9b2b2e370 100644
--- a/src/backend/commands/repack.c
+++ b/src/backend/commands/repack.c
@@ -279,6 +279,21 @@ ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel)
/* Determine the lock mode to use. */
lockmode = RepackLockLevel((params.options & CLUOPT_CONCURRENT) != 0);
+ if ((params.options & CLUOPT_CONCURRENT) != 0)
+ {
+ /*
+ * Make sure we're not in a transaction block.
+ *
+ * The reason is that repack_setup_logical_decoding() could wait
+ * indefinitely for our XID to complete. (The deadlock detector would
+ * not recognize it because we'd be waiting for ourselves, i.e. no
+ * real lock conflict.) It would be possible to run in a transaction
+ * block if we had no XID, but this restriction is simpler for users
+ * to understand and we don't lose any functionality.
+ */
+ PreventInTransactionBlock(isTopLevel, "REPACK (CONCURRENTLY)");
+ }
+
/*
* If a single relation is specified, process it and we're done ... unless
* the relation is a partitioned table, in which case we fall through.
@@ -511,19 +526,7 @@ cluster_rel(RepackCommand cmd, Relation OldHeap, Oid indexOid,
* replica index OID.
*/
if (concurrent)
- {
- /*
- * Make sure we're not in a transaction block.
- *
- * The reason is that repack_setup_logical_decoding() could deadlock
- * if there's an XID already assigned. It would be possible to run in
- * a transaction block if we had no XID, but this restriction is
- * simpler for users to understand and we don't lose anything.
- */
- PreventInTransactionBlock(isTopLevel, "REPACK (CONCURRENTLY)");
-
check_repack_concurrently_requirements(OldHeap, &ident_idx);
- }
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
--
2.47.3
--qr3jlalmmcpkiodg
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
filename="v53-0006-Error-out-any-process-that-would-block-at-REPACK.patch"
^ permalink raw reply [nested|flat] 55+ messages in thread
* [PATCH v52 07/10] Check for transaction block early in ExecRepack
@ 2026-04-03 15:23 Srinath Reddy Sadipiralla <[email protected]>
0 siblings, 0 replies; 55+ messages in thread
From: Srinath Reddy Sadipiralla @ 2026-04-03 15:23 UTC (permalink / raw)
Currently, executing REPACK (CONCURRENTLY) without a table name inside a
transaction block throws the error "REPACK CONCURRENTLY requires explicit
table name" instead of the expected transaction block error. This occurs
because ExecRepack() validates the parsed options and missing relation
before verifying the transaction state.
This behavior is inconsistent with other utility commands like VACUUM
,REINDEX, etc; which invoke PreventInTransactionBlock() at the very start
of their execution to properly reject execution inside user transactions
before validating targets.
Add PreventInTransactionBlock to the top of ExecRepack() to enforce the
transaction block restriction early. This prevents the user from fixing
a missing table error only to immediately hit a transaction block error,
and also ensures consistency with rest of the commands.
Author: Srinath Reddy Sadipiralla <[email protected]>
---
src/backend/commands/repack.c | 26 ++++++++++++--------------
1 file changed, 12 insertions(+), 14 deletions(-)
diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c
index d6e446d582d..d4c1f0e7652 100644
--- a/src/backend/commands/repack.c
+++ b/src/backend/commands/repack.c
@@ -292,6 +292,18 @@ ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel)
MyProc->statusFlags |= PROC_IN_CONCURRENT_REPACK;
ProcGlobal->statusFlags[MyProc->pgxactoff] = MyProc->statusFlags;
LWLockRelease(ProcArrayLock);
+
+ /*
+ * Make sure we're not in a transaction block.
+ *
+ * The reason is that repack_setup_logical_decoding() could wait
+ * indefinitely for our XID to complete. (The deadlock detector would
+ * not recognize it because we'd be waiting for ourselves, i.e. no
+ * real lock conflict.) It would be possible to run in a transaction
+ * block if we had no XID, but this restriction is simpler for users
+ * to understand and we don't lose anything.
+ */
+ PreventInTransactionBlock(isTopLevel, "REPACK (CONCURRENTLY)");
}
/*
@@ -523,21 +535,7 @@ cluster_rel(RepackCommand cmd, Relation OldHeap, Oid indexOid,
* replica index OID.
*/
if (concurrent)
- {
- /*
- * Make sure we're not in a transaction block.
- *
- * The reason is that repack_setup_logical_decoding() could wait
- * indefinitely for our XID to complete. (The deadlock detector would
- * not recognize it because we'd be waiting for ourselves, i.e. no
- * real lock conflict.) It would be possible to run in a transaction
- * block if we had no XID, but this restriction is simpler for users
- * to understand and we don't lose anything.
- */
- PreventInTransactionBlock(isTopLevel, "REPACK (CONCURRENTLY)");
-
check_repack_concurrently_requirements(OldHeap, &ident_idx);
- }
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
--
2.47.3
--gp2pyozrd5pweboh
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
filename="v52-0008-Introduce-an-option-to-make-logical-replication-.patch"
^ permalink raw reply [nested|flat] 55+ messages in thread
* [PATCH v52 07/10] Check for transaction block early in ExecRepack
@ 2026-04-03 15:23 Srinath Reddy Sadipiralla <[email protected]>
0 siblings, 0 replies; 55+ messages in thread
From: Srinath Reddy Sadipiralla @ 2026-04-03 15:23 UTC (permalink / raw)
Currently, executing REPACK (CONCURRENTLY) without a table name inside a
transaction block throws the error "REPACK CONCURRENTLY requires explicit
table name" instead of the expected transaction block error. This occurs
because ExecRepack() validates the parsed options and missing relation
before verifying the transaction state.
This behavior is inconsistent with other utility commands like VACUUM
,REINDEX, etc; which invoke PreventInTransactionBlock() at the very start
of their execution to properly reject execution inside user transactions
before validating targets.
Add PreventInTransactionBlock to the top of ExecRepack() to enforce the
transaction block restriction early. This prevents the user from fixing
a missing table error only to immediately hit a transaction block error,
and also ensures consistency with rest of the commands.
Author: Srinath Reddy Sadipiralla <[email protected]>
---
src/backend/commands/repack.c | 26 ++++++++++++--------------
1 file changed, 12 insertions(+), 14 deletions(-)
diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c
index d6e446d582d..d4c1f0e7652 100644
--- a/src/backend/commands/repack.c
+++ b/src/backend/commands/repack.c
@@ -292,6 +292,18 @@ ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel)
MyProc->statusFlags |= PROC_IN_CONCURRENT_REPACK;
ProcGlobal->statusFlags[MyProc->pgxactoff] = MyProc->statusFlags;
LWLockRelease(ProcArrayLock);
+
+ /*
+ * Make sure we're not in a transaction block.
+ *
+ * The reason is that repack_setup_logical_decoding() could wait
+ * indefinitely for our XID to complete. (The deadlock detector would
+ * not recognize it because we'd be waiting for ourselves, i.e. no
+ * real lock conflict.) It would be possible to run in a transaction
+ * block if we had no XID, but this restriction is simpler for users
+ * to understand and we don't lose anything.
+ */
+ PreventInTransactionBlock(isTopLevel, "REPACK (CONCURRENTLY)");
}
/*
@@ -523,21 +535,7 @@ cluster_rel(RepackCommand cmd, Relation OldHeap, Oid indexOid,
* replica index OID.
*/
if (concurrent)
- {
- /*
- * Make sure we're not in a transaction block.
- *
- * The reason is that repack_setup_logical_decoding() could wait
- * indefinitely for our XID to complete. (The deadlock detector would
- * not recognize it because we'd be waiting for ourselves, i.e. no
- * real lock conflict.) It would be possible to run in a transaction
- * block if we had no XID, but this restriction is simpler for users
- * to understand and we don't lose anything.
- */
- PreventInTransactionBlock(isTopLevel, "REPACK (CONCURRENTLY)");
-
check_repack_concurrently_requirements(OldHeap, &ident_idx);
- }
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
--
2.47.3
--gp2pyozrd5pweboh
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
filename="v52-0008-Introduce-an-option-to-make-logical-replication-.patch"
^ permalink raw reply [nested|flat] 55+ messages in thread
end of thread, other threads:[~2026-04-03 15:23 UTC | newest]
Thread overview: 55+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2019-09-26 21:05 [PATCH v2 5/8] jit: explain: remove backend lifetime module count from function name. Andres Freund <[email protected]>
2019-09-26 21:05 [PATCH v1 05/12] jit: explain: remove backend lifetime module count from function name. Andres Freund <[email protected]>
2024-10-10 15:54 Re: type cache cleanup improvements Artur Zakirov <[email protected]>
2024-10-13 12:08 ` Re: type cache cleanup improvements Alexander Korotkov <[email protected]>
2024-10-15 07:34 ` Re: type cache cleanup improvements jian he <[email protected]>
2024-10-15 08:08 ` Re: type cache cleanup improvements Alexander Korotkov <[email protected]>
2024-10-15 08:45 ` Re: type cache cleanup improvements Artur Zakirov <[email protected]>
2024-10-15 09:50 ` Re: type cache cleanup improvements jian he <[email protected]>
2024-10-15 13:16 ` Re: type cache cleanup improvements Artur Zakirov <[email protected]>
2024-10-20 17:47 ` Re: type cache cleanup improvements Alexander Korotkov <[email protected]>
2024-10-20 18:00 ` Re: type cache cleanup improvements Alexander Korotkov <[email protected]>
2024-10-20 22:09 ` Re: type cache cleanup improvements Alexander Korotkov <[email protected]>
2024-10-20 23:32 ` Re: type cache cleanup improvements Dagfinn Ilmari Mannsåker <[email protected]>
2024-10-21 05:40 ` Re: type cache cleanup improvements Andrei Lepikhov <[email protected]>
2024-10-21 08:10 ` Re: type cache cleanup improvements Alexander Korotkov <[email protected]>
2024-10-21 10:16 ` Re: type cache cleanup improvements Dagfinn Ilmari Mannsåker <[email protected]>
2024-10-21 11:30 ` Re: type cache cleanup improvements Alexander Korotkov <[email protected]>
2024-10-22 07:34 ` Re: type cache cleanup improvements Alexander Korotkov <[email protected]>
2024-10-22 15:09 ` Re: type cache cleanup improvements Pavel Borisov <[email protected]>
2024-10-22 17:33 ` Re: type cache cleanup improvements Alexander Korotkov <[email protected]>
2024-10-25 08:35 ` Re: type cache cleanup improvements Andres Freund <[email protected]>
2024-10-25 09:48 ` Re: type cache cleanup improvements Alexander Korotkov <[email protected]>
2024-10-25 10:13 ` Re: type cache cleanup improvements Alexander Korotkov <[email protected]>
2025-04-11 20:32 ` Re: type cache cleanup improvements Noah Misch <[email protected]>
2025-04-11 21:43 ` Re: type cache cleanup improvements Alexander Korotkov <[email protected]>
2025-04-21 01:54 ` Re: type cache cleanup improvements Alexander Korotkov <[email protected]>
2025-04-29 00:56 ` Re: type cache cleanup improvements Noah Misch <[email protected]>
2025-04-29 10:00 ` Re: type cache cleanup improvements Alexander Korotkov <[email protected]>
2024-10-21 07:51 ` Re: type cache cleanup improvements jian he <[email protected]>
2024-10-21 08:11 ` Re: type cache cleanup improvements Alexander Korotkov <[email protected]>
2024-10-17 09:41 ` Re: type cache cleanup improvements Andrei Lepikhov <[email protected]>
2024-10-20 17:36 ` Re: type cache cleanup improvements Alexander Korotkov <[email protected]>
2024-10-21 05:36 ` Re: type cache cleanup improvements Andrei Lepikhov <[email protected]>
2025-03-07 00:05 Re: Add Pipelining support in psql Jelte Fennema-Nio <[email protected]>
2025-03-07 08:08 ` Re: Add Pipelining support in psql Anthonin Bonnefoy <[email protected]>
2025-03-07 22:31 ` Re: Add Pipelining support in psql Daniel Verite <[email protected]>
2025-03-17 09:50 ` Re: Add Pipelining support in psql Anthonin Bonnefoy <[email protected]>
2025-03-17 10:19 ` Re: Add Pipelining support in psql Michael Paquier <[email protected]>
2025-03-18 00:50 ` Re: Add Pipelining support in psql Michael Paquier <[email protected]>
2025-03-18 08:55 ` Re: Add Pipelining support in psql Anthonin Bonnefoy <[email protected]>
2025-03-18 09:27 ` Re: Add Pipelining support in psql Jelte Fennema-Nio <[email protected]>
2025-03-19 02:28 ` Re: Add Pipelining support in psql Michael Paquier <[email protected]>
2025-03-19 13:05 ` Re: Add Pipelining support in psql Daniel Verite <[email protected]>
2025-03-19 04:49 ` Re: Add Pipelining support in psql Michael Paquier <[email protected]>
2025-03-18 09:36 ` Re: Add Pipelining support in psql Daniel Verite <[email protected]>
2025-03-18 23:56 ` Re: Add Pipelining support in psql Michael Paquier <[email protected]>
2025-04-16 14:31 Re: Add Pipelining support in psql a.kozhemyakin <[email protected]>
2025-04-16 16:18 ` Re: Add Pipelining support in psql Michael Paquier <[email protected]>
2025-04-22 12:37 ` Re: Add Pipelining support in psql Anthonin Bonnefoy <[email protected]>
2026-04-03 15:23 [PATCH v51 07/10] Check for transaction block early in ExecRepack Srinath Reddy Sadipiralla <[email protected]>
2026-04-03 15:23 [PATCH v53 5/7] Check for transaction block early in ExecRepack Srinath Reddy Sadipiralla <[email protected]>
2026-04-03 15:23 [PATCH v51 07/10] Check for transaction block early in ExecRepack Srinath Reddy Sadipiralla <[email protected]>
2026-04-03 15:23 [PATCH v53 5/7] Check for transaction block early in ExecRepack Srinath Reddy Sadipiralla <[email protected]>
2026-04-03 15:23 [PATCH v52 07/10] Check for transaction block early in ExecRepack Srinath Reddy Sadipiralla <[email protected]>
2026-04-03 15:23 [PATCH v52 07/10] Check for transaction block early in ExecRepack Srinath Reddy Sadipiralla <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox