public inbox for [email protected]
help / color / mirror / Atom feed[PATCH v1 05/12] jit: explain: remove backend lifetime module count from function name.
41+ messages / 8 participants
[nested] [flat]
* [PATCH v1 05/12] jit: explain: remove backend lifetime module count from function name.
@ 2019-09-26 21:05 Andres Freund <[email protected]>
0 siblings, 0 replies; 41+ messages in thread
From: Andres Freund @ 2019-09-26 21:05 UTC (permalink / raw)
Also expand function name to include in which module the function is -
without that it's harder to analyze which functions were emitted
separately (a performance concern).
Author:
Reviewed-By:
Discussion: https://postgr.es/m/
Backpatch:
---
src/backend/commands/explain.c | 65 +++++++++++++++++++++++++++++-----
src/backend/jit/llvm/llvmjit.c | 18 +++++++---
src/include/jit/llvmjit.h | 5 ++-
3 files changed, 75 insertions(+), 13 deletions(-)
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 3ccb76bdfd1..02455865d9f 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -2228,6 +2228,43 @@ show_expression(Node *node, ExprState *expr, const char *qlabel,
}
}
+/*
+ * To make JIT explain output reproducible, remove the module generation from
+ * function names. That makes it a bit harder to correlate with profiles etc,
+ * but reproducability is more important.
+ */
+static char *
+jit_funcname_for_display(const char *funcname)
+{
+ int func_counter; /* nth function in query */
+ size_t mod_num; /* nth module in query */
+ size_t mod_generation; /* nth module in backend */
+ int basename_end;
+ int matchcount = 0;
+
+ /*
+ * The pattern we need to match, see llvm_expand_funcname, is
+ * "%s_%zu_%d_mod_%zu". Find the fourth _ from the end, so a _ in the name
+ * is OK.
+ */
+ for (basename_end = strlen(funcname); basename_end >= 0; basename_end--)
+ {
+ if (funcname[basename_end] == '_' && ++matchcount == 4)
+ break;
+ }
+
+ /* couldn't parse, bail out */
+ if (matchcount != 4)
+ return pstrdup(funcname);
+
+ /* couldn't parse, bail out */
+ if (sscanf(funcname + basename_end, "_%zu_%d_mod_%zu",
+ &mod_num, &func_counter, &mod_generation) != 3)
+ return pstrdup(funcname);
+
+ return psprintf("%s_%zu_%d", pnstrdup(funcname, basename_end), mod_num, func_counter);
+}
+
static void
show_jit_expr_details(ExprState *expr, ExplainState *es)
{
@@ -2239,7 +2276,8 @@ show_jit_expr_details(ExprState *expr, ExplainState *es)
if (es->format == EXPLAIN_FORMAT_TEXT)
{
if (expr->flags & EEO_FLAG_JIT_EXPR)
- appendStringInfo(es->str, "JIT-Expr: %s", expr->expr_funcname);
+ appendStringInfo(es->str, "JIT-Expr: %s",
+ jit_funcname_for_display(expr->expr_funcname));
else
appendStringInfoString(es->str, "JIT-Expr: false");
@@ -2250,19 +2288,22 @@ show_jit_expr_details(ExprState *expr, ExplainState *es)
*/
if (expr->scan_funcname)
- appendStringInfo(es->str, ", JIT-Deform-Scan: %s", expr->scan_funcname);
+ appendStringInfo(es->str, ", JIT-Deform-Scan: %s",
+ jit_funcname_for_display(expr->scan_funcname));
else if (expr->flags & EEO_FLAG_JIT_EXPR &&
expr->flags & EEO_FLAG_DEFORM_SCAN)
appendStringInfo(es->str, ", JIT-Deform-Scan: false");
if (expr->outer_funcname)
- appendStringInfo(es->str, ", JIT-Deform-Outer: %s", expr->outer_funcname);
+ appendStringInfo(es->str, ", JIT-Deform-Outer: %s",
+ jit_funcname_for_display(expr->outer_funcname));
else if (expr->flags & EEO_FLAG_JIT_EXPR &&
expr->flags & EEO_FLAG_DEFORM_OUTER)
appendStringInfo(es->str, ", JIT-Deform-Outer: false");
if (expr->inner_funcname)
- appendStringInfo(es->str, ", JIT-Deform-Inner: %s", expr->inner_funcname);
+ appendStringInfo(es->str, ", JIT-Deform-Inner: %s",
+ jit_funcname_for_display(expr->inner_funcname));
else if (expr->flags & EEO_FLAG_JIT_EXPR &&
expr->flags & (EEO_FLAG_DEFORM_INNER))
appendStringInfo(es->str, ", JIT-Deform-Inner: false");
@@ -2270,26 +2311,34 @@ show_jit_expr_details(ExprState *expr, ExplainState *es)
else
{
if (expr->flags & EEO_FLAG_JIT_EXPR)
- ExplainPropertyText("JIT-Expr", expr->expr_funcname, es);
+ ExplainPropertyText("JIT-Expr",
+ jit_funcname_for_display(expr->expr_funcname),
+ es);
else
ExplainPropertyBool("JIT-Expr", false, es);
if (expr->scan_funcname)
- ExplainProperty("JIT-Deform-Scan", NULL, expr->scan_funcname, false, es);
+ ExplainProperty("JIT-Deform-Scan", NULL,
+ jit_funcname_for_display(expr->scan_funcname),
+ false, es);
else if (expr->flags & EEO_FLAG_DEFORM_SCAN)
ExplainProperty("JIT-Deform-Scan", NULL, "false", true, es);
else
ExplainProperty("JIT-Deform-Scan", NULL, "null", true, es);
if (expr->outer_funcname)
- ExplainProperty("JIT-Deform-Outer", NULL, expr->outer_funcname, false, es);
+ ExplainProperty("JIT-Deform-Outer", NULL,
+ jit_funcname_for_display(expr->outer_funcname),
+ false, es);
else if (expr->flags & EEO_FLAG_DEFORM_OUTER)
ExplainProperty("JIT-Deform-Outer", NULL, "false", true, es);
else
ExplainProperty("JIT-Deform-Outer", NULL, "null", true, es);
if (expr->inner_funcname)
- ExplainProperty("JIT-Deform-Inner", NULL, expr->inner_funcname, false, es);
+ ExplainProperty("JIT-Deform-Inner", NULL,
+ jit_funcname_for_display(expr->inner_funcname),
+ false, es);
else if (expr->flags & EEO_FLAG_DEFORM_INNER)
ExplainProperty("JIT-Deform-Inner", NULL, "false", true, es);
else
diff --git a/src/backend/jit/llvm/llvmjit.c b/src/backend/jit/llvm/llvmjit.c
index 5489e118041..177a00f3826 100644
--- a/src/backend/jit/llvm/llvmjit.c
+++ b/src/backend/jit/llvm/llvmjit.c
@@ -227,6 +227,8 @@ llvm_mutable_module(LLVMJitContext *context)
char *
llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
{
+ char *funcname;
+
Assert(context->module != NULL);
context->base.instr.created_functions++;
@@ -234,11 +236,19 @@ llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
/*
* Previously we used dots to separate, but turns out some tools, e.g.
* GDB, don't like that and truncate name.
+ *
+ * Append the backend-lifetime module count to the end, so it's easier for
+ * humans and machines to compare the generated function names across
+ * queries, the prefix will be the same from query execution to query
+ * execution.
*/
- return psprintf("%s_%zu_%d",
- basename,
- context->module_generation,
- context->counter++);
+ funcname = psprintf("%s_%zu_%d_mod_%zu",
+ basename,
+ context->base.instr.created_modules - 1,
+ context->counter++,
+ context->module_generation);
+
+ return funcname;
}
/*
diff --git a/src/include/jit/llvmjit.h b/src/include/jit/llvmjit.h
index 6178864b2e6..e45ff99194f 100644
--- a/src/include/jit/llvmjit.h
+++ b/src/include/jit/llvmjit.h
@@ -41,7 +41,10 @@ typedef struct LLVMJitContext
{
JitContext base;
- /* number of modules created */
+ /*
+ * llvm_generation when ->module was created, monotonically increasing
+ * within the lifetime of a backend.
+ */
size_t module_generation;
/* current, "open for write", module */
--
2.23.0.162.gf1d4a28250
--kgppz6muakthis74
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v1-0006-WIP-explain-Show-per-phase-information-about-aggr.patch"
^ permalink raw reply [nested|flat] 41+ messages in thread
* [PATCH v2 5/8] jit: explain: remove backend lifetime module count from function name.
@ 2019-09-26 21:05 Andres Freund <[email protected]>
0 siblings, 0 replies; 41+ messages in thread
From: Andres Freund @ 2019-09-26 21:05 UTC (permalink / raw)
Also expand function name to include in which module the function is -
without that it's harder to analyze which functions were emitted
separately (a performance concern).
Author:
Reviewed-By:
Discussion: https://postgr.es/m/
Backpatch:
---
src/backend/commands/explain.c | 65 +++++++++++++++++++++++++++++-----
src/backend/jit/llvm/llvmjit.c | 18 +++++++---
src/include/jit/llvmjit.h | 5 ++-
3 files changed, 75 insertions(+), 13 deletions(-)
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 3ccb76bdfd1..02455865d9f 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -2228,6 +2228,43 @@ show_expression(Node *node, ExprState *expr, const char *qlabel,
}
}
+/*
+ * To make JIT explain output reproducible, remove the module generation from
+ * function names. That makes it a bit harder to correlate with profiles etc,
+ * but reproducability is more important.
+ */
+static char *
+jit_funcname_for_display(const char *funcname)
+{
+ int func_counter; /* nth function in query */
+ size_t mod_num; /* nth module in query */
+ size_t mod_generation; /* nth module in backend */
+ int basename_end;
+ int matchcount = 0;
+
+ /*
+ * The pattern we need to match, see llvm_expand_funcname, is
+ * "%s_%zu_%d_mod_%zu". Find the fourth _ from the end, so a _ in the name
+ * is OK.
+ */
+ for (basename_end = strlen(funcname); basename_end >= 0; basename_end--)
+ {
+ if (funcname[basename_end] == '_' && ++matchcount == 4)
+ break;
+ }
+
+ /* couldn't parse, bail out */
+ if (matchcount != 4)
+ return pstrdup(funcname);
+
+ /* couldn't parse, bail out */
+ if (sscanf(funcname + basename_end, "_%zu_%d_mod_%zu",
+ &mod_num, &func_counter, &mod_generation) != 3)
+ return pstrdup(funcname);
+
+ return psprintf("%s_%zu_%d", pnstrdup(funcname, basename_end), mod_num, func_counter);
+}
+
static void
show_jit_expr_details(ExprState *expr, ExplainState *es)
{
@@ -2239,7 +2276,8 @@ show_jit_expr_details(ExprState *expr, ExplainState *es)
if (es->format == EXPLAIN_FORMAT_TEXT)
{
if (expr->flags & EEO_FLAG_JIT_EXPR)
- appendStringInfo(es->str, "JIT-Expr: %s", expr->expr_funcname);
+ appendStringInfo(es->str, "JIT-Expr: %s",
+ jit_funcname_for_display(expr->expr_funcname));
else
appendStringInfoString(es->str, "JIT-Expr: false");
@@ -2250,19 +2288,22 @@ show_jit_expr_details(ExprState *expr, ExplainState *es)
*/
if (expr->scan_funcname)
- appendStringInfo(es->str, ", JIT-Deform-Scan: %s", expr->scan_funcname);
+ appendStringInfo(es->str, ", JIT-Deform-Scan: %s",
+ jit_funcname_for_display(expr->scan_funcname));
else if (expr->flags & EEO_FLAG_JIT_EXPR &&
expr->flags & EEO_FLAG_DEFORM_SCAN)
appendStringInfo(es->str, ", JIT-Deform-Scan: false");
if (expr->outer_funcname)
- appendStringInfo(es->str, ", JIT-Deform-Outer: %s", expr->outer_funcname);
+ appendStringInfo(es->str, ", JIT-Deform-Outer: %s",
+ jit_funcname_for_display(expr->outer_funcname));
else if (expr->flags & EEO_FLAG_JIT_EXPR &&
expr->flags & EEO_FLAG_DEFORM_OUTER)
appendStringInfo(es->str, ", JIT-Deform-Outer: false");
if (expr->inner_funcname)
- appendStringInfo(es->str, ", JIT-Deform-Inner: %s", expr->inner_funcname);
+ appendStringInfo(es->str, ", JIT-Deform-Inner: %s",
+ jit_funcname_for_display(expr->inner_funcname));
else if (expr->flags & EEO_FLAG_JIT_EXPR &&
expr->flags & (EEO_FLAG_DEFORM_INNER))
appendStringInfo(es->str, ", JIT-Deform-Inner: false");
@@ -2270,26 +2311,34 @@ show_jit_expr_details(ExprState *expr, ExplainState *es)
else
{
if (expr->flags & EEO_FLAG_JIT_EXPR)
- ExplainPropertyText("JIT-Expr", expr->expr_funcname, es);
+ ExplainPropertyText("JIT-Expr",
+ jit_funcname_for_display(expr->expr_funcname),
+ es);
else
ExplainPropertyBool("JIT-Expr", false, es);
if (expr->scan_funcname)
- ExplainProperty("JIT-Deform-Scan", NULL, expr->scan_funcname, false, es);
+ ExplainProperty("JIT-Deform-Scan", NULL,
+ jit_funcname_for_display(expr->scan_funcname),
+ false, es);
else if (expr->flags & EEO_FLAG_DEFORM_SCAN)
ExplainProperty("JIT-Deform-Scan", NULL, "false", true, es);
else
ExplainProperty("JIT-Deform-Scan", NULL, "null", true, es);
if (expr->outer_funcname)
- ExplainProperty("JIT-Deform-Outer", NULL, expr->outer_funcname, false, es);
+ ExplainProperty("JIT-Deform-Outer", NULL,
+ jit_funcname_for_display(expr->outer_funcname),
+ false, es);
else if (expr->flags & EEO_FLAG_DEFORM_OUTER)
ExplainProperty("JIT-Deform-Outer", NULL, "false", true, es);
else
ExplainProperty("JIT-Deform-Outer", NULL, "null", true, es);
if (expr->inner_funcname)
- ExplainProperty("JIT-Deform-Inner", NULL, expr->inner_funcname, false, es);
+ ExplainProperty("JIT-Deform-Inner", NULL,
+ jit_funcname_for_display(expr->inner_funcname),
+ false, es);
else if (expr->flags & EEO_FLAG_DEFORM_INNER)
ExplainProperty("JIT-Deform-Inner", NULL, "false", true, es);
else
diff --git a/src/backend/jit/llvm/llvmjit.c b/src/backend/jit/llvm/llvmjit.c
index 5489e118041..177a00f3826 100644
--- a/src/backend/jit/llvm/llvmjit.c
+++ b/src/backend/jit/llvm/llvmjit.c
@@ -227,6 +227,8 @@ llvm_mutable_module(LLVMJitContext *context)
char *
llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
{
+ char *funcname;
+
Assert(context->module != NULL);
context->base.instr.created_functions++;
@@ -234,11 +236,19 @@ llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
/*
* Previously we used dots to separate, but turns out some tools, e.g.
* GDB, don't like that and truncate name.
+ *
+ * Append the backend-lifetime module count to the end, so it's easier for
+ * humans and machines to compare the generated function names across
+ * queries, the prefix will be the same from query execution to query
+ * execution.
*/
- return psprintf("%s_%zu_%d",
- basename,
- context->module_generation,
- context->counter++);
+ funcname = psprintf("%s_%zu_%d_mod_%zu",
+ basename,
+ context->base.instr.created_modules - 1,
+ context->counter++,
+ context->module_generation);
+
+ return funcname;
}
/*
diff --git a/src/include/jit/llvmjit.h b/src/include/jit/llvmjit.h
index 6178864b2e6..e45ff99194f 100644
--- a/src/include/jit/llvmjit.h
+++ b/src/include/jit/llvmjit.h
@@ -41,7 +41,10 @@ typedef struct LLVMJitContext
{
JitContext base;
- /* number of modules created */
+ /*
+ * llvm_generation when ->module was created, monotonically increasing
+ * within the lifetime of a backend.
+ */
size_t module_generation;
/* current, "open for write", module */
--
2.23.0.385.gbc12974a89
--ga6shgqrocqphdjc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0006-WIP-explain-Show-per-phase-information-about-aggr.patch"
^ permalink raw reply [nested|flat] 41+ messages in thread
* [PATCH 15/16] Rename header comment and #define
@ 2022-11-19 16:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 41+ messages in thread
From: Justin Pryzby @ 2022-11-19 16:31 UTC (permalink / raw)
Maybe should fix more of these?
find src/ contrib/ -name '*.h' |xargs awk '/^#ifndef.*_H$/ && FILENAME!~/-/{reg=toupper(gensub(".*/","",1,FILENAME)); if ($0!~reg){print reg,$0}}'
---
src/backend/utils/misc/guc_internal.h | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/src/backend/utils/misc/guc_internal.h b/src/backend/utils/misc/guc_internal.h
index be3940951a2..bd96405cdad 100644
--- a/src/backend/utils/misc/guc_internal.h
+++ b/src/backend/utils/misc/guc_internal.h
@@ -1,16 +1,16 @@
/*--------------------------------------------------------------------
- * guc_internals.h
+ * guc_internal.h
*
* Declarations shared between backend/utils/misc/guc.c and
* backend/utils/misc/guc-file.l
*
* Copyright (c) 2000-2022, PostgreSQL Global Development Group
*
- * src/include/utils/guc_internals.h
+ * src/include/utils/guc_internal.h
*--------------------------------------------------------------------
*/
-#ifndef GUC_INTERNALS_H
-#define GUC_INTERNALS_H
+#ifndef GUC_INTERNAL_H
+#define GUC_INTERNAL_H
#include "utils/guc.h"
@@ -23,4 +23,4 @@ extern void record_config_file_error(const char *errmsg,
ConfigVariable **head_p,
ConfigVariable **tail_p);
-#endif /* GUC_INTERNALS_H */
+#endif /* GUC_INTERNAL_H */
--
2.25.1
--TBNym+cBXeFsS4Vs
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0016-Add-some-missing-newlines-after-function-definitions.patch"
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2024-12-10 10:43 Anthonin Bonnefoy <[email protected]>
0 siblings, 1 reply; 41+ messages in thread
From: Anthonin Bonnefoy @ 2024-12-10 10:43 UTC (permalink / raw)
To: Jelte Fennema-Nio <[email protected]>; +Cc: Michael Paquier <[email protected]>; pgsql-hackers
An improved version with simplifications and refinements.
num_queries (2nd element in the pipeline status prompt) is now used to
track queued queries that were not flushed (with a flush request or
sync) to the server. It used to count both unflushed queries and
flushed queries.
Code in ExecQueryAndProcessResults should be simpler now.
- DiscardAbortedPipelineResults function handles both discarding of
results until a synchronisation point is reached or discarding of
results until there's no more pending results.
- The logic to process the pipeline's results and getting the next
results fit more with the existing flow.
- Tests didn't cover chunk results so I've added additional tests to
cover use of pipelining + FETCH_COUNT
Attachments:
[application/octet-stream] v04-0001-Add-pipelining-support-in-psql.patch (48.3K, ../../CAO6_XqrDPpaHqvKA0r8kfuHBE8BZ5ZqWLwucu30cXobeGtvBMA@mail.gmail.com/2-v04-0001-Add-pipelining-support-in-psql.patch)
download | inline diff:
From 6c3b0a842505b8814e04d1dba55eb0abee8c3b09 Mon Sep 17 00:00:00 2001
From: Anthonin Bonnefoy <[email protected]>
Date: Tue, 5 Nov 2024 10:26:54 +0100
Subject: Add pipelining support in psql
With \bind, \parse, \bind_named and \close, it is possible to issue
queries from psql using the extended protocol. However, it wasn't
possible to send those queries using pipelining and the only way to test
pipelined queries was through pgbench's tap tests.
This patch adds additional psql meta-commands to support pipelining:
\startpipeline, \endpipeline and \syncpipeline, mirroring the existing
meta-commands in pgbench. Additional meta-commands allow to flush and
read results of an ongoing pipeline: \flushrequest, \flush and \getresults
\startpipeline starts a new pipeline. All extended queries will be
queued until the end of the pipeline is reached.
\endpipeline ends an ongoing pipeline. All queued commands will be sent
to the server and all responses will be processed by the psql.
\syncpipeline queues a synchronisation point without flushing the
commands to the server
\flush Call PQflush on psql's connection
\flushrequest queues a flushrequest
\getresults reads server's results. Unsent data are automatically pushed
when \getresults is called
Those meta-commands will allow to test pipeline behaviour using
psql regression tests.
---
doc/src/sgml/ref/psql-ref.sgml | 77 ++++
src/bin/psql/command.c | 135 +++++++
src/bin/psql/common.c | 205 ++++++++++-
src/bin/psql/help.c | 6 +
src/bin/psql/prompt.c | 12 +
src/bin/psql/settings.h | 13 +-
src/bin/psql/tab-complete.in.c | 8 +-
src/test/regress/expected/psql.out | 547 +++++++++++++++++++++++++++++
src/test/regress/sql/psql.sql | 337 ++++++++++++++++++
9 files changed, 1330 insertions(+), 10 deletions(-)
diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml
index e42073ed748..34cd4e2b767 100644
--- a/doc/src/sgml/ref/psql-ref.sgml
+++ b/doc/src/sgml/ref/psql-ref.sgml
@@ -3562,6 +3562,67 @@ testdb=> <userinput>\setenv LESS -imx4F</userinput>
</listitem>
</varlistentry>
+ <varlistentry id="app-psql-meta-command-pipeline">
+ <term><literal>\startpipeline</literal></term>
+ <term><literal>\syncpipeline</literal></term>
+ <term><literal>\endpipeline</literal></term>
+ <term><literal>\flushrequest</literal></term>
+ <term><literal>\flush</literal></term>
+ <term><literal>\getresults</literal></term>
+
+ <listitem>
+ <para>
+ This group of commands implements pipelining of SQL statements.
+ A pipeline must begin with a <command>\startpipeline</command>
+ and end with an <command>\endpipeline</command>. In between there
+ may be any number of <command>\syncpipeline</command> commands,
+ which sends a <link linkend="protocol-flow-ext-query">sync message</link>
+ without ending the ongoing pipeline and flushing the send buffer.
+ In pipeline mode, statements are sent to the server without waiting
+ for the results of previous statements. See
+ <xref linkend="libpq-pipeline-mode"/> for more details.
+ </para>
+
+ <para>
+ Pipeline mode requires the use of extended query protocol. All queries need
+ to be sent using the meta-commands <literal>\bind</literal>,
+ <literal>\bind_named</literal>, <literal>\close</literal> or
+ <literal>\parse</literal>. While a pipeline is ongoing,
+ <literal>\g</literal> will append the current query buffer to the pipeline and
+ other meta-commands like <literal>\gx</literal> or <literal>\gdesc</literal>
+ are not allowed in pipeline mode.
+ </para>
+
+ <para>
+ <command>\flushrequest</command> appends a flush command to the pipeline,
+ allowing to read results with <command>\getresults</command> without issuing
+ a sync or ending the pipeline. <command>\getresults</command> will automatically
+ push unsent data to the server. <command>\flush</command> can be used to manually
+ push unsent data.
+ </para>
+
+ <para>
+ </para>
+
+ <para>
+ Example:
+<programlisting>
+\startpipeline
+SELECT 1 \bind \g
+SELECT $1 \parse stmt1
+\bind_named stmt1 1 \g
+SELECT pg_current_xact_id() \bind \g
+\flushrequest
+\getresults
+\syncpipeline
+SELECT pg_current_xact_id() \bind \g
+\close stmt1
+\endpipeline
+</programlisting></para>
+
+ </listitem>
+ </varlistentry>
+
<varlistentry id="app-psql-meta-command-t-lc">
<term><literal>\t</literal></term>
@@ -4705,6 +4766,22 @@ testdb=> <userinput>INSERT INTO my_table VALUES (:'content');</userinput>
</listitem>
</varlistentry>
+ <varlistentry id="app-psql-prompting-p-uc">
+ <term><literal>%P</literal></term>
+ <listitem>
+ <para>
+ Pipeline status: an empty string when not in a pipeline,
+ or <literal>|num_syncs,num_queries,pending_results|</literal>
+ when in an ongoing pipeline, or <literal>+num_syncs,num_queries,pending_results+</literal>
+ when in an aborted pipeline. <literal>num_syncs</literal> is the number
+ of syncs queued in the pipeline, <literal>num_queries</literal>
+ is the number of queries queued in the pipeline and <literal>pending_results</literal>
+ is the number of queries sent to the server with a <command>\flushrequest</command>
+ or a <command>\syncpipeline</command> that can be retrieved with <command>\getresults</command>.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry id="app-psql-prompting-r">
<term><literal>%R</literal></term>
<listitem>
diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c
index 1f3cbb11f7c..411f6a6cd71 100644
--- a/src/bin/psql/command.c
+++ b/src/bin/psql/command.c
@@ -90,9 +90,12 @@ static backslashResult exec_command_else(PsqlScanState scan_state, ConditionalSt
PQExpBuffer query_buf);
static backslashResult exec_command_endif(PsqlScanState scan_state, ConditionalStack cstack,
PQExpBuffer query_buf);
+static backslashResult exec_command_endpipeline(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_encoding(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_errverbose(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_f(PsqlScanState scan_state, bool active_branch);
+static backslashResult exec_command_flush(PsqlScanState scan_state, bool active_branch);
+static backslashResult exec_command_flushrequest(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_g(PsqlScanState scan_state, bool active_branch,
const char *cmd);
static backslashResult process_command_g_options(char *first_option,
@@ -103,6 +106,7 @@ static backslashResult exec_command_gdesc(PsqlScanState scan_state, bool active_
static backslashResult exec_command_getenv(PsqlScanState scan_state, bool active_branch,
const char *cmd);
static backslashResult exec_command_gexec(PsqlScanState scan_state, bool active_branch);
+static backslashResult exec_command_getresults(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_gset(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_help(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_html(PsqlScanState scan_state, bool active_branch);
@@ -132,6 +136,8 @@ static backslashResult exec_command_setenv(PsqlScanState scan_state, bool active
const char *cmd);
static backslashResult exec_command_sf_sv(PsqlScanState scan_state, bool active_branch,
const char *cmd, bool is_func);
+static backslashResult exec_command_startpipeline(PsqlScanState scan_state, bool active_branch);
+static backslashResult exec_command_syncpipeline(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_t(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_T(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_timing(PsqlScanState scan_state, bool active_branch);
@@ -351,18 +357,26 @@ exec_command(const char *cmd,
status = exec_command_else(scan_state, cstack, query_buf);
else if (strcmp(cmd, "endif") == 0)
status = exec_command_endif(scan_state, cstack, query_buf);
+ else if (strcmp(cmd, "endpipeline") == 0)
+ status = exec_command_endpipeline(scan_state, active_branch);
else if (strcmp(cmd, "encoding") == 0)
status = exec_command_encoding(scan_state, active_branch);
else if (strcmp(cmd, "errverbose") == 0)
status = exec_command_errverbose(scan_state, active_branch);
else if (strcmp(cmd, "f") == 0)
status = exec_command_f(scan_state, active_branch);
+ else if (strcmp(cmd, "flush") == 0)
+ status = exec_command_flush(scan_state, active_branch);
+ else if (strcmp(cmd, "flushrequest") == 0)
+ status = exec_command_flushrequest(scan_state, active_branch);
else if (strcmp(cmd, "g") == 0 || strcmp(cmd, "gx") == 0)
status = exec_command_g(scan_state, active_branch, cmd);
else if (strcmp(cmd, "gdesc") == 0)
status = exec_command_gdesc(scan_state, active_branch);
else if (strcmp(cmd, "getenv") == 0)
status = exec_command_getenv(scan_state, active_branch, cmd);
+ else if (strcmp(cmd, "getresults") == 0)
+ status = exec_command_getresults(scan_state, active_branch);
else if (strcmp(cmd, "gexec") == 0)
status = exec_command_gexec(scan_state, active_branch);
else if (strcmp(cmd, "gset") == 0)
@@ -408,6 +422,10 @@ exec_command(const char *cmd,
status = exec_command_sf_sv(scan_state, active_branch, cmd, true);
else if (strcmp(cmd, "sv") == 0 || strcmp(cmd, "sv+") == 0)
status = exec_command_sf_sv(scan_state, active_branch, cmd, false);
+ else if (strcmp(cmd, "startpipeline") == 0)
+ status = exec_command_startpipeline(scan_state, active_branch);
+ else if (strcmp(cmd, "syncpipeline") == 0)
+ status = exec_command_syncpipeline(scan_state, active_branch);
else if (strcmp(cmd, "t") == 0)
status = exec_command_t(scan_state, active_branch);
else if (strcmp(cmd, "T") == 0)
@@ -1491,6 +1509,38 @@ exec_command_f(PsqlScanState scan_state, bool active_branch)
return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR;
}
+/*
+ * \flush -- call PQflush on the connection
+ */
+static backslashResult
+exec_command_flush(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_FLUSH;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
+/*
+ * \flushrequest -- send a flush request to the server
+ */
+static backslashResult
+exec_command_flushrequest(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_SEND_FLUSH_REQUEST;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
/*
* \g [(pset-option[=pset-value] ...)] [filename/shell-command]
* \gx [(pset-option[=pset-value] ...)] [filename/shell-command]
@@ -1526,6 +1576,13 @@ exec_command_g(PsqlScanState scan_state, bool active_branch, const char *cmd)
if (status == PSQL_CMD_SKIP_LINE && active_branch)
{
+ if (strcmp(cmd, "gx") == 0 && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pg_log_error("\\gx not allowed in pipeline mode");
+ clean_extended_state();
+ return PSQL_CMD_ERROR;
+ }
+
if (!fname)
pset.gfname = NULL;
else
@@ -1679,6 +1736,23 @@ exec_command_getenv(PsqlScanState scan_state, bool active_branch,
return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR;
}
+/*
+ * \getresults -- read results
+ */
+static backslashResult
+exec_command_getresults(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_GET_RESULTS;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
+
/*
* \gexec -- send query and execute each field of result
*/
@@ -1689,6 +1763,12 @@ exec_command_gexec(PsqlScanState scan_state, bool active_branch)
if (active_branch)
{
+ if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pg_log_error("\\gexec not allowed in pipeline mode");
+ clean_extended_state();
+ return PSQL_CMD_ERROR;
+ }
pset.gexec_flag = true;
status = PSQL_CMD_SEND;
}
@@ -1709,6 +1789,13 @@ exec_command_gset(PsqlScanState scan_state, bool active_branch)
char *prefix = psql_scan_slash_option(scan_state,
OT_NORMAL, NULL, false);
+ if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pg_log_error("\\gset not allowed in pipeline mode");
+ clean_extended_state();
+ return PSQL_CMD_ERROR;
+ }
+
if (prefix)
pset.gset_prefix = prefix;
else
@@ -2672,6 +2759,54 @@ exec_command_sf_sv(PsqlScanState scan_state, bool active_branch,
return status;
}
+/*
+ * \startpipeline -- enter pipeline mode
+ */
+static backslashResult
+exec_command_startpipeline(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_START_PIPELINE_MODE;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
+/*
+ * \syncpipeline -- send a sync message to an active pipeline
+ */
+static backslashResult
+exec_command_syncpipeline(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_SEND_PIPELINE_SYNC;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
+/*
+ * \endpipeline -- end pipeline mode
+ */
+static backslashResult
+exec_command_endpipeline(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_END_PIPELINE_MODE;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
/*
* \t -- turn off table headers and row count
*/
diff --git a/src/bin/psql/common.c b/src/bin/psql/common.c
index 8a9211db41a..7a225a4fbad 100644
--- a/src/bin/psql/common.c
+++ b/src/bin/psql/common.c
@@ -415,10 +415,12 @@ AcceptResult(const PGresult *result, bool show_error)
case PGRES_EMPTY_QUERY:
case PGRES_COPY_IN:
case PGRES_COPY_OUT:
+ case PGRES_PIPELINE_SYNC:
/* Fine, do nothing */
OK = true;
break;
+ case PGRES_PIPELINE_ABORTED:
case PGRES_BAD_RESPONSE:
case PGRES_NONFATAL_ERROR:
case PGRES_FATAL_ERROR:
@@ -1050,6 +1052,7 @@ PrintQueryResult(PGresult *result, bool last,
success = true;
break;
+ case PGRES_PIPELINE_ABORTED:
case PGRES_BAD_RESPONSE:
case PGRES_NONFATAL_ERROR:
case PGRES_FATAL_ERROR:
@@ -1418,6 +1421,48 @@ DescribeQuery(const char *query, double *elapsed_msec)
return OK;
}
+/*
+ * Read and discard all results in an aborted pipeline.
+ *
+ * If a synchronisation point is found, we can stop discarding results as
+ * pipeline will switch back to an OK state. If no synchronisation point
+ * is available, we need to stop when there's no more pending results,
+ * otherwise, calling PQgetResults will block.
+ */
+static PGresult *
+discardAbortedPipelineResults(void)
+{
+ for (;;)
+ {
+ PGresult *res = PQgetResult(pset.db);
+ ExecStatusType result_status = PQresultStatus(res);
+
+ if (result_status == PGRES_PIPELINE_SYNC)
+ /* Found a synchronisation point */
+ return res;
+ else if (res == NULL)
+ /* A query was processed, decrement the counters */
+ pset.pending_results--;
+ else if (res != NULL && result_status == PGRES_FATAL_ERROR)
+ return res;
+
+ if (pset.pending_results == 0 && pset.num_syncs == 0)
+
+ /*
+ * There's no more results to get and there's no synchronisation
+ * point to stop at. This will leave the pipeline in an aborted
+ * state.
+ */
+ return res;
+
+ /*
+ * An aborted pipeline will have either NULL results or results in an
+ * PGRES_PIPELINE_ABORTED status
+ */
+ Assert(res == NULL || result_status == PGRES_PIPELINE_ABORTED);
+ PQclear(res);
+ }
+}
/*
* ExecQueryAndProcessResults: utility function for use by SendQuery()
@@ -1451,6 +1496,8 @@ ExecQueryAndProcessResults(const char *query,
bool timing = pset.timing;
bool success = false;
bool return_early = false;
+ bool process_pipeline = false;
+ bool get_pending_results = false;
instr_time before,
after;
PGresult *result;
@@ -1466,9 +1513,13 @@ ExecQueryAndProcessResults(const char *query,
{
case PSQL_SEND_EXTENDED_CLOSE:
success = PQsendClosePrepared(pset.db, pset.stmtName);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ pset.num_queries++;
break;
case PSQL_SEND_EXTENDED_PARSE:
success = PQsendPrepare(pset.db, pset.stmtName, query, 0, NULL);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ pset.num_queries++;
break;
case PSQL_SEND_EXTENDED_QUERY_PARAMS:
Assert(pset.stmtName == NULL);
@@ -1476,6 +1527,8 @@ ExecQueryAndProcessResults(const char *query,
pset.bind_nparams, NULL,
(const char *const *) pset.bind_params,
NULL, NULL, 0);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ pset.num_queries++;
break;
case PSQL_SEND_EXTENDED_QUERY_PREPARED:
Assert(pset.stmtName != NULL);
@@ -1483,6 +1536,62 @@ ExecQueryAndProcessResults(const char *query,
pset.bind_nparams,
(const char *const *) pset.bind_params,
NULL, NULL, 0);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ pset.num_queries++;
+ break;
+ case PSQL_START_PIPELINE_MODE:
+ success = PQenterPipelineMode(pset.db);
+ break;
+ case PSQL_END_PIPELINE_MODE:
+ success = PQpipelineSync(pset.db);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ /*
+ * End of the pipeline, all queued commands need to be
+ * processed
+ */
+ process_pipeline = true;
+ pset.num_syncs++;
+ pset.pending_results += pset.num_queries;
+ pset.num_queries = 0;
+ }
+ break;
+ case PSQL_SEND_PIPELINE_SYNC:
+ success = PQsendPipelineSync(pset.db);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pset.num_syncs++;
+ pset.pending_results += pset.num_queries;
+ pset.num_queries = 0;
+ }
+ break;
+ case PSQL_FLUSH:
+ success = PQflush(pset.db);
+ break;
+ case PSQL_SEND_FLUSH_REQUEST:
+ success = PQsendFlushRequest(pset.db);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pset.pending_results += pset.num_queries;
+ pset.num_queries = 0;
+ }
+ break;
+ case PSQL_GET_RESULTS:
+ if (pset.pending_results == 0)
+ {
+ /*
+ * If no sync or flush request were sent, PQgetResult will
+ * block. Forbid the call to \getresults to avoid staying
+ * stuck
+ */
+ pg_log_info("No pending results to get");
+ success = false;
+ }
+ else
+ {
+ get_pending_results = true;
+ success = true;
+ }
break;
case PSQL_SEND_QUERY:
success = PQsendQuery(pset.db, query);
@@ -1501,6 +1610,16 @@ ExecQueryAndProcessResults(const char *query,
return -1;
}
+ if (!get_pending_results && !process_pipeline &&
+ PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ /*
+ * We're in a pipeline and haven't reached the pipeline end or there
+ * was no request to read pipeline results, exit.
+ */
+ return 1;
+ }
+
/*
* Fetch the result in chunks if FETCH_COUNT is set, except when:
*
@@ -1548,7 +1667,7 @@ ExecQueryAndProcessResults(const char *query,
{
ExecStatusType result_status;
bool is_chunked_result = false;
- PGresult *next_result;
+ PGresult *next_result = NULL;
bool last;
if (!AcceptResult(result, false))
@@ -1571,6 +1690,9 @@ ExecQueryAndProcessResults(const char *query,
ClearOrSaveResult(result);
success = false;
+ if (result_status == PGRES_PIPELINE_ABORTED)
+ pg_log_info("Pipeline aborted, command didn't run");
+
/*
* switch to next result
*/
@@ -1585,6 +1707,15 @@ ExecQueryAndProcessResults(const char *query,
* ignore manually.
*/
result = NULL;
+ else if (process_pipeline || get_pending_results)
+
+ /*
+ * We have an error within a pipeline. All commands are
+ * aborted until the next synchronisation point. We need to
+ * consume all results until this synchronisation point, or
+ * stop when there's no more result to discard
+ */
+ result = discardAbortedPipelineResults();
else
result = PQgetResult(pset.db);
@@ -1771,12 +1902,59 @@ ExecQueryAndProcessResults(const char *query,
}
}
+ if (result_status == PGRES_PIPELINE_SYNC)
+ {
+ Assert(pset.num_syncs > 0);
+ /* We have a sync response, decrease the sync counter */
+ pset.num_syncs--;
+
+ /*
+ * After a synchronisation point, reset success state to print
+ * possible successful results that will be processed after this
+ */
+ success = true;
+
+ /*
+ * If all syncs were processed and pipeline end was requested,
+ * exit pipeline mode
+ */
+ if (process_pipeline && pset.num_syncs == 0)
+ success &= PQexitPipelineMode(pset.db);
+ }
+ else if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF &&
+ result_status != PGRES_PIPELINE_SYNC)
+
+ /*
+ * We are in a pipeline and have a non sync response, decrease the
+ * pending results counter
+ */
+ pset.pending_results--;
+
/*
* Check PQgetResult() again. In the typical case of a single-command
* string, it will return NULL. Otherwise, we'll have other results
* to process. We need to do that to check whether this is the last.
*/
- next_result = PQgetResult(pset.db);
+ if (PQpipelineStatus(pset.db) == PQ_PIPELINE_OFF)
+ next_result = PQgetResult(pset.db);
+ else
+ {
+ /*
+ * In pipeline mode, a NULL result indicates the end of the
+ * current query being processed. Call PQgetResult to consume this
+ * NULL
+ */
+ if (result_status != PGRES_PIPELINE_SYNC)
+ {
+ next_result = PQgetResult(pset.db);
+ Assert(next_result == NULL);
+ }
+
+ /* We can now get the next result in the pipeline */
+ if (pset.num_syncs > 0 || pset.pending_results > 0)
+ next_result = PQgetResult(pset.db);
+ }
+
last = (next_result == NULL);
/*
@@ -1798,8 +1976,12 @@ ExecQueryAndProcessResults(const char *query,
*elapsed_msec = INSTR_TIME_GET_MILLISEC(after);
}
- /* this may or may not print something depending on settings */
- if (result != NULL)
+ /*
+ * This may or may not print something depending on settings. A
+ * pipeline sync will have a non null result but doesn't have anything
+ * to print, thus ignore them
+ */
+ if (result != NULL && result_status != PGRES_PIPELINE_SYNC)
{
/*
* If results need to be printed into the file specified by \g,
@@ -1837,6 +2019,15 @@ ExecQueryAndProcessResults(const char *query,
/* close \g file if we opened it */
CloseGOutput(gfile_fout, gfile_is_pipe);
+ if (process_pipeline)
+ {
+ /* After a pipeline is processed, pipeline num_syncs should be 0 */
+ Assert(pset.num_syncs == 0);
+ /* and all queries were processed */
+ Assert(pset.num_queries == 0);
+ }
+ Assert(pset.pending_results == 0);
+
/* may need this to recover from conn loss during COPY */
if (!CheckConnection())
return -1;
@@ -2296,7 +2487,13 @@ clean_extended_state(void)
free(pset.stmtName);
pset.bind_params = NULL;
break;
+ case PSQL_START_PIPELINE_MODE: /* \startpipeline */
+ case PSQL_END_PIPELINE_MODE: /* \endpipeline */
+ case PSQL_SEND_PIPELINE_SYNC: /* \syncpipeline */
case PSQL_SEND_QUERY:
+ case PSQL_FLUSH: /* \flush */
+ case PSQL_GET_RESULTS: /* \getresults */
+ case PSQL_SEND_FLUSH_REQUEST: /* \flushrequest */
break;
}
diff --git a/src/bin/psql/help.c b/src/bin/psql/help.c
index 3f4afc2d141..d15ff6d40c3 100644
--- a/src/bin/psql/help.c
+++ b/src/bin/psql/help.c
@@ -167,15 +167,21 @@ slashUsage(unsigned short int pager)
HELP0(" \\close STMT_NAME close an existing prepared statement\n");
HELP0(" \\copyright show PostgreSQL usage and distribution terms\n");
HELP0(" \\crosstabview [COLUMNS] execute query and display result in crosstab\n");
+ HELP0(" \\endpipeline exit pipeline mode\n");
HELP0(" \\errverbose show most recent error message at maximum verbosity\n");
+ HELP0(" \\flush push unsent data to the server\n");
+ HELP0(" \\flushrequest send a flushrequest command\n");
HELP0(" \\g [(OPTIONS)] [FILE] execute query (and send result to file or |pipe);\n"
" \\g with no arguments is equivalent to a semicolon\n");
HELP0(" \\gdesc describe result of query, without executing it\n");
+ HELP0(" \\getresults read pending results\n");
HELP0(" \\gexec execute query, then execute each value in its result\n");
HELP0(" \\gset [PREFIX] execute query and store result in psql variables\n");
HELP0(" \\gx [(OPTIONS)] [FILE] as \\g, but forces expanded output mode\n");
HELP0(" \\parse STMT_NAME create a prepared statement\n");
HELP0(" \\q quit psql\n");
+ HELP0(" \\startpipeline enter pipeline mode\n");
+ HELP0(" \\syncpipeline add a synchronisation point to an ongoing pipeline\n");
HELP0(" \\watch [[i=]SEC] [c=N] [m=MIN]\n"
" execute query every SEC seconds, up to N times,\n"
" stop if less than MIN rows are returned\n");
diff --git a/src/bin/psql/prompt.c b/src/bin/psql/prompt.c
index 0d99d00ac92..126a6252c56 100644
--- a/src/bin/psql/prompt.c
+++ b/src/bin/psql/prompt.c
@@ -31,6 +31,9 @@
* sockets, "[local:/dir/name]" if not default
* %m - like %M, but hostname only (before first dot), or always "[local]"
* %p - backend pid
+ * %P - pipeline status: no pipeline: empty
+ * ongoing pipeline: |num_syncs,num_queries,pending_results|
+ * aborted pipeline: +num_syncs,num_queries,pending_results+
* %> - database server port number
* %n - database user name
* %/ - current database
@@ -175,7 +178,16 @@ get_prompt(promptStatus_t status, ConditionalStack cstack)
snprintf(buf, sizeof(buf), "%d", pid);
}
break;
+ case 'P':
+ {
+ PGpipelineStatus status = PQpipelineStatus(pset.db);
+ if (status == PQ_PIPELINE_ON)
+ snprintf(buf, sizeof(buf), "|%d,%d,%d|", pset.num_syncs, pset.num_queries, pset.pending_results);
+ else if (status == PQ_PIPELINE_ABORTED)
+ snprintf(buf, sizeof(buf), "+%d,%d,%d+", pset.num_syncs, pset.num_queries, pset.pending_results);
+ break;
+ }
case '0':
case '1':
case '2':
diff --git a/src/bin/psql/settings.h b/src/bin/psql/settings.h
index a22de8ef78e..400a8a9d878 100644
--- a/src/bin/psql/settings.h
+++ b/src/bin/psql/settings.h
@@ -23,8 +23,8 @@
#define DEFAULT_EDITOR_LINENUMBER_ARG "+"
#endif
-#define DEFAULT_PROMPT1 "%/%R%x%# "
-#define DEFAULT_PROMPT2 "%/%R%x%# "
+#define DEFAULT_PROMPT1 "%/%R%P%x%# "
+#define DEFAULT_PROMPT2 "%/%R%P%x%# "
#define DEFAULT_PROMPT3 ">> "
/*
@@ -69,6 +69,12 @@ typedef enum
PSQL_SEND_EXTENDED_PARSE,
PSQL_SEND_EXTENDED_QUERY_PARAMS,
PSQL_SEND_EXTENDED_QUERY_PREPARED,
+ PSQL_SEND_PIPELINE_SYNC,
+ PSQL_START_PIPELINE_MODE,
+ PSQL_END_PIPELINE_MODE,
+ PSQL_FLUSH,
+ PSQL_SEND_FLUSH_REQUEST,
+ PSQL_GET_RESULTS,
} PSQL_SEND_MODE;
typedef enum
@@ -111,6 +117,9 @@ typedef struct _psqlSettings
char **bind_params; /* parameters for extended query protocol call */
char *stmtName; /* prepared statement name used for extended
* query protocol commands */
+ int pending_results;/* number of results available to read */
+ int num_queries; /* number of piped queries */
+ int num_syncs; /* number of piped syncs */
bool crosstab_flag; /* one-shot request to crosstab result */
char *ctv_args[4]; /* \crosstabview arguments */
diff --git a/src/bin/psql/tab-complete.in.c b/src/bin/psql/tab-complete.in.c
index bbd08770c3d..de7240d8690 100644
--- a/src/bin/psql/tab-complete.in.c
+++ b/src/bin/psql/tab-complete.in.c
@@ -1867,9 +1867,9 @@ psql_completion(const char *text, int start, int end)
"\\drds", "\\drg", "\\dRs", "\\dRp", "\\ds",
"\\dt", "\\dT", "\\dv", "\\du", "\\dx", "\\dX", "\\dy",
"\\echo", "\\edit", "\\ef", "\\elif", "\\else", "\\encoding",
- "\\endif", "\\errverbose", "\\ev",
- "\\f",
- "\\g", "\\gdesc", "\\getenv", "\\gexec", "\\gset", "\\gx",
+ "\\endif", "\\endpipeline", "\\errverbose", "\\ev",
+ "\\f", "\\flush", "\\flushrequest",
+ "\\g", "\\gdesc", "\\getenv", "\\getresults", "\\gexec", "\\gset", "\\gx",
"\\help", "\\html",
"\\if", "\\include", "\\include_relative", "\\ir",
"\\list", "\\lo_import", "\\lo_export", "\\lo_list", "\\lo_unlink",
@@ -1877,7 +1877,7 @@ psql_completion(const char *text, int start, int end)
"\\parse", "\\password", "\\print", "\\prompt", "\\pset",
"\\qecho", "\\quit",
"\\reset",
- "\\s", "\\set", "\\setenv", "\\sf", "\\sv",
+ "\\s", "\\set", "\\setenv", "\\sf", "\\startpipeline", "\\sv", "\\syncpipeline",
"\\t", "\\T", "\\timing",
"\\unset",
"\\x",
diff --git a/src/test/regress/expected/psql.out b/src/test/regress/expected/psql.out
index 36dc31c16c4..81ae2176e44 100644
--- a/src/test/regress/expected/psql.out
+++ b/src/test/regress/expected/psql.out
@@ -6828,3 +6828,550 @@ CREATE TABLE defprivs (a int);
\pset null ''
DROP TABLE defprivs;
+-- pipelining
+CREATE TABLE psql_pipeline(a INTEGER PRIMARY KEY, s TEXT);
+-- single query
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+-- multiple queries
+\startpipeline
+SELECT $1 \bind 'val1' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val2 | val3
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val2 | val3
+(1 row)
+
+-- send multiple syncs
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\syncpipeline
+\syncpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\syncpipeline
+SELECT $1, $2 \bind 'val4' 'val5' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val2 | val3
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val4 | val5
+(1 row)
+
+-- startpipeline shouldn't have any effect if already in a pipeline
+\startpipeline
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+-- Convert an implicit tx block to an explicit tx block
+\startpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 2 \g
+ROLLBACK \bind \g
+\endpipeline
+-- Multiple explicit transactions
+\startpipeline
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+ROLLBACK \bind \g
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+COMMIT \bind \g
+\endpipeline
+-- COPY FROM STDIN
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+-- COPY FROM STDIN with \flushrequest + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\flushrequest
+\getresults
+ ?column?
+----------
+ val1
+(1 row)
+
+message type 0x5a arrived from server while idle
+\endpipeline
+-- COPY FROM STDIN with \syncpipeline + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\syncpipeline
+\getresults
+ ?column?
+----------
+ val1
+(1 row)
+
+\endpipeline
+-- COPY TO STDOUT
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+1 \N
+2 test2
+3 test3
+4 test4
+-- COPY TO STDOUT with \flushrequest + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\flushrequest
+\getresults
+ ?column?
+----------
+ val1
+(1 row)
+
+1 \N
+2 test2
+3 test3
+4 test4
+\endpipeline
+-- COPY TO STDOUT with \syncpipeline + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\syncpipeline
+\getresults
+ ?column?
+----------
+ val1
+(1 row)
+
+1 \N
+2 test2
+3 test3
+4 test4
+\endpipeline
+-- Use \parse and \bind_named
+\startpipeline
+SELECT $1 \parse ''
+SELECT $1, $2 \parse ''
+SELECT $2 \parse pipeline_1
+\bind_named '' 1 2 \g
+\bind_named pipeline_1 2 \g
+\endpipeline
+ERROR: could not determine data type of parameter $1
+-- \getresults displays all results preceding a \flushrequest
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+\flushrequest
+\getresults
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+\endpipeline
+-- \getresults displays all results preceding a sync
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+\syncpipeline
+\getresults
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+\endpipeline
+-- \getresults immediately returns if there's no result to fetch
+\startpipeline
+\getresults
+No pending results to get
+SELECT $1 \bind 2 \g
+\getresults
+No pending results to get
+\flushrequest
+\endpipeline
+ ?column?
+----------
+ 2
+(1 row)
+
+\getresults
+No pending results to get
+-- \getresults only fetch results preceding a flushrequest
+\startpipeline
+SELECT $1 \bind 2 \g
+\flushrequest
+SELECT $1 \bind 2 \g
+\getresults
+ ?column?
+----------
+ 2
+(1 row)
+
+\endpipeline
+ ?column?
+----------
+ 2
+(1 row)
+
+-- same for sync request
+\startpipeline
+SELECT $1 \bind 2 \g
+\syncpipeline
+SELECT $1 \bind 2 \g
+\getresults
+ ?column?
+----------
+ 2
+(1 row)
+
+\endpipeline
+ ?column?
+----------
+ 2
+(1 row)
+
+-- use pipeline with chunked results with both \getresults and \endpipeline
+\startpipeline
+\set FETCH_COUNT 10
+SELECT $1 \bind 2 \g
+\flushrequest
+\getresults
+ ?column?
+----------
+ 2
+(1 row)
+
+SELECT $1 \bind 2 \g
+\endpipeline
+ ?column?
+----------
+ 2
+(1 row)
+
+\unset FETCH_COUNT
+-- pipelining errors
+-- endpipeline outside of pipeline should fail
+\endpipeline
+cannot send pipeline when not in pipeline mode
+-- Query using simple protocol should not be sent and should leave the pipeline usable
+\startpipeline
+SELECT 1;
+PQsendQuery not allowed in pipeline mode
+SELECT $1 \bind 'val1' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+-- After an aborted pipeline, commands after a sync should be displayed
+\startpipeline
+SELECT $1 \bind \g
+\syncpipeline
+SELECT $1 \bind 1 \g
+\endpipeline
+ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
+ ?column?
+----------
+ 1
+(1 row)
+
+-- Incorrect number of parameters, the pipeline will be aborted and following queries won't be executed
+\startpipeline
+SELECT \bind 'val1' \g
+SELECT $1 \bind 'val1' \g
+\endpipeline
+ERROR: bind message supplies 1 parameters, but prepared statement "" requires 0
+-- An explicit transaction with an error needs to be rollbacked after the pipeline
+\startpipeline
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+ROLLBACK \bind \g
+\endpipeline
+ERROR: duplicate key value violates unique constraint "psql_pipeline_pkey"
+DETAIL: Key (a)=(1) already exists.
+ROLLBACK;
+-- \watch sends a simple query which won't be allowed within a pipeline
+\startpipeline
+SELECT \bind \g
+\watch 1
+PQsendQuery not allowed in pipeline mode
+
+\endpipeline
+--
+(1 row)
+
+-- \gdesc should fail as synchronous commands are not allowed in pipeline, pipeline should still be usable
+\startpipeline
+SELECT $1 \bind 1 \gdesc
+synchronous command execution functions are not allowed in pipeline mode
+SELECT $1 \bind 1 \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+-- \gset is not allowed, pipeline should still be usable
+\startpipeline
+SELECT $1 as i, $2 as j \parse ''
+SELECT $1 as k, $2 as l \parse 'second'
+\bind_named '' 1 2 \gset
+\gset not allowed in pipeline mode
+\bind_named second 1 2 \gset pref02_ \echo :pref02_i :pref02_j
+\gset not allowed in pipeline mode
+\bind_named '' 1 2 \g
+\endpipeline
+ i | j
+---+---
+ 1 | 2
+(1 row)
+
+-- \gx is not allowed, pipeline should still be usable
+\startpipeline
+SELECT $1 \bind 1 \gx
+\gx not allowed in pipeline mode
+\reset
+SELECT $1 \bind 1 \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+-- \gx warning should be emitted in an aborted pipeline
+\startpipeline
+SELECT $1 \bind \g
+\flushrequest
+\getresults
+ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
+SELECT $1 \bind 1 \gx
+\gx not allowed in pipeline mode
+\endpipeline
+-- \gexec is not allowed, pipeline should still be usable
+\startpipeline
+SELECT 'INSERT INTO psql_pipeline(a) SELECT generate_series(1, 10)' \parse 'insert_stmt'
+\bind_named insert_stmt \gexec
+\gexec not allowed in pipeline mode
+\bind_named insert_stmt \g
+SELECT COUNT(*) FROM psql_pipeline \bind \g
+\endpipeline
+ ?column?
+------------------------------------------------------------
+ INSERT INTO psql_pipeline(a) SELECT generate_series(1, 10)
+(1 row)
+
+ count
+-------
+ 4
+(1 row)
+
+-- After an error, pipeline is aborted and requires a sync to be reusable
+\startpipeline
+SELECT $1 \bind \g
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+\flushrequest
+\getresults
+ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
+-- Pipeline is aborted
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+-- Sync allows pipeline to recover
+\syncpipeline
+\getresults
+Pipeline aborted, command didn't run
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+\flushrequest
+\getresults
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 1
+(1 row)
+
+\endpipeline
+-- Test chunked results with an aborted pipeline
+\startpipeline
+\set FETCH_COUNT 10
+SELECT $1 \bind \g
+\flushrequest
+\getresults
+ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
+SELECT $1 \bind \g
+\endpipeline
+fetching results in chunked mode failed
+Pipeline aborted, command didn't run
+\unset FETCH_COUNT
+-- pipelining and transaction block behaviour
+-- set local will issue a warning when modifying a GUC outside of a transaction block
+-- The change will still be valid as a pipeline runs within an implicit transaction block
+-- Sending a sync will commit the implicit transaction block. The first command after a sync
+-- won't be seen as belonging to a pipeline.
+\startpipeline
+SET LOCAL statement_timeout='1h' \bind \g
+SHOW statement_timeout \bind \g
+\syncpipeline
+SHOW statement_timeout \bind \g
+SET LOCAL statement_timeout='2h' \bind \g
+SHOW statement_timeout \bind \g
+\endpipeline
+WARNING: SET LOCAL can only be used in transaction blocks
+ statement_timeout
+-------------------
+ 1h
+(1 row)
+
+ statement_timeout
+-------------------
+ 0
+(1 row)
+
+ statement_timeout
+-------------------
+ 2h
+(1 row)
+
+-- Reindex concurrently is forbidden in the middle of a pipeline
+\startpipeline
+SELECT $1 \bind 1 \g
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ERROR: REINDEX CONCURRENTLY cannot run inside a transaction block
+-- Reindex concurrently will work if it's the first command of a pipeline
+\startpipeline
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ ?column?
+----------
+ 2
+(1 row)
+
+-- subtransactions are not allowed in pipeline mode
+\startpipeline
+SAVEPOINT a \bind \g
+SELECT $1 \bind 1 \g
+ROLLBACK TO SAVEPOINT a \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ERROR: SAVEPOINT can only be used in transaction blocks
+-- Lock command will fail as first pipeline command is not seen as a transaction block
+\startpipeline
+LOCK psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ERROR: LOCK TABLE can only be used in transaction blocks
+-- Lock command will succeed after the first command as pipeline will be seen as an implicit transaction block
+\startpipeline
+SELECT $1 \bind 1 \g
+LOCK psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+-- Vacuum command will work as the first command
+\startpipeline
+VACUUM psql_pipeline \bind \g
+\endpipeline
+-- Vacuum command will fail within pipeline implicit transaction
+\startpipeline
+SELECT 1 \bind \g
+VACUUM psql_pipeline \bind \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ERROR: VACUUM cannot run inside a transaction block
+-- Vacuum command will work after a sync
+\startpipeline
+SELECT 1 \bind \g
+\syncpipeline
+VACUUM psql_pipeline \bind \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
diff --git a/src/test/regress/sql/psql.sql b/src/test/regress/sql/psql.sql
index c5021fc0b13..075c0a873ca 100644
--- a/src/test/regress/sql/psql.sql
+++ b/src/test/regress/sql/psql.sql
@@ -1929,3 +1929,340 @@ CREATE TABLE defprivs (a int);
\z defprivs
\pset null ''
DROP TABLE defprivs;
+
+-- pipelining
+CREATE TABLE psql_pipeline(a INTEGER PRIMARY KEY, s TEXT);
+
+-- single query
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\endpipeline
+
+-- multiple queries
+\startpipeline
+SELECT $1 \bind 'val1' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\endpipeline
+
+-- send multiple syncs
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\syncpipeline
+\syncpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\syncpipeline
+SELECT $1, $2 \bind 'val4' 'val5' \g
+\endpipeline
+
+-- startpipeline shouldn't have any effect if already in a pipeline
+\startpipeline
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\endpipeline
+
+-- Convert an implicit tx block to an explicit tx block
+\startpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 2 \g
+ROLLBACK \bind \g
+\endpipeline
+
+-- Multiple explicit transactions
+\startpipeline
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+ROLLBACK \bind \g
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+COMMIT \bind \g
+\endpipeline
+
+-- COPY FROM STDIN
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\endpipeline
+2 test2
+\.
+
+-- COPY FROM STDIN with \flushrequest + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\flushrequest
+\getresults
+3 test3
+\.
+\endpipeline
+
+-- COPY FROM STDIN with \syncpipeline + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\syncpipeline
+\getresults
+4 test4
+\.
+\endpipeline
+
+-- COPY TO STDOUT
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\endpipeline
+
+-- COPY TO STDOUT with \flushrequest + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\flushrequest
+\getresults
+\endpipeline
+
+-- COPY TO STDOUT with \syncpipeline + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\syncpipeline
+\getresults
+\endpipeline
+
+-- Use \parse and \bind_named
+\startpipeline
+SELECT $1 \parse ''
+SELECT $1, $2 \parse ''
+SELECT $2 \parse pipeline_1
+\bind_named '' 1 2 \g
+\bind_named pipeline_1 2 \g
+\endpipeline
+
+-- \getresults displays all results preceding a \flushrequest
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+\flushrequest
+\getresults
+\endpipeline
+
+-- \getresults displays all results preceding a sync
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+\syncpipeline
+\getresults
+\endpipeline
+
+-- \getresults immediately returns if there's no result to fetch
+\startpipeline
+\getresults
+SELECT $1 \bind 2 \g
+\getresults
+\flushrequest
+\endpipeline
+\getresults
+
+-- \getresults only fetch results preceding a flushrequest
+\startpipeline
+SELECT $1 \bind 2 \g
+\flushrequest
+SELECT $1 \bind 2 \g
+\getresults
+\endpipeline
+
+-- same for sync request
+\startpipeline
+SELECT $1 \bind 2 \g
+\syncpipeline
+SELECT $1 \bind 2 \g
+\getresults
+\endpipeline
+
+-- use pipeline with chunked results with both \getresults and \endpipeline
+\startpipeline
+\set FETCH_COUNT 10
+SELECT $1 \bind 2 \g
+\flushrequest
+\getresults
+SELECT $1 \bind 2 \g
+\endpipeline
+\unset FETCH_COUNT
+
+-- pipelining errors
+
+-- endpipeline outside of pipeline should fail
+\endpipeline
+
+-- Query using simple protocol should not be sent and should leave the pipeline usable
+\startpipeline
+SELECT 1;
+SELECT $1 \bind 'val1' \g
+\endpipeline
+
+-- After an aborted pipeline, commands after a sync should be displayed
+\startpipeline
+SELECT $1 \bind \g
+\syncpipeline
+SELECT $1 \bind 1 \g
+\endpipeline
+
+-- Incorrect number of parameters, the pipeline will be aborted and following queries won't be executed
+\startpipeline
+SELECT \bind 'val1' \g
+SELECT $1 \bind 'val1' \g
+\endpipeline
+
+-- An explicit transaction with an error needs to be rollbacked after the pipeline
+\startpipeline
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+ROLLBACK \bind \g
+\endpipeline
+ROLLBACK;
+
+-- \watch sends a simple query which won't be allowed within a pipeline
+\startpipeline
+SELECT \bind \g
+\watch 1
+\endpipeline
+
+-- \gdesc should fail as synchronous commands are not allowed in pipeline, pipeline should still be usable
+\startpipeline
+SELECT $1 \bind 1 \gdesc
+SELECT $1 \bind 1 \g
+\endpipeline
+
+-- \gset is not allowed, pipeline should still be usable
+\startpipeline
+SELECT $1 as i, $2 as j \parse ''
+SELECT $1 as k, $2 as l \parse 'second'
+\bind_named '' 1 2 \gset
+\bind_named second 1 2 \gset pref02_ \echo :pref02_i :pref02_j
+\bind_named '' 1 2 \g
+\endpipeline
+
+-- \gx is not allowed, pipeline should still be usable
+\startpipeline
+SELECT $1 \bind 1 \gx
+\reset
+SELECT $1 \bind 1 \g
+\endpipeline
+
+-- \gx warning should be emitted in an aborted pipeline
+\startpipeline
+SELECT $1 \bind \g
+\flushrequest
+\getresults
+SELECT $1 \bind 1 \gx
+\endpipeline
+
+-- \gexec is not allowed, pipeline should still be usable
+\startpipeline
+SELECT 'INSERT INTO psql_pipeline(a) SELECT generate_series(1, 10)' \parse 'insert_stmt'
+\bind_named insert_stmt \gexec
+\bind_named insert_stmt \g
+SELECT COUNT(*) FROM psql_pipeline \bind \g
+\endpipeline
+
+-- After an error, pipeline is aborted and requires a sync to be reusable
+\startpipeline
+SELECT $1 \bind \g
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+\flushrequest
+\getresults
+-- Pipeline is aborted
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+-- Sync allows pipeline to recover
+\syncpipeline
+\getresults
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+\flushrequest
+\getresults
+\endpipeline
+
+-- Test chunked results with an aborted pipeline
+\startpipeline
+\set FETCH_COUNT 10
+SELECT $1 \bind \g
+\flushrequest
+\getresults
+SELECT $1 \bind \g
+\endpipeline
+\unset FETCH_COUNT
+
+-- pipelining and transaction block behaviour
+
+-- set local will issue a warning when modifying a GUC outside of a transaction block
+-- The change will still be valid as a pipeline runs within an implicit transaction block
+-- Sending a sync will commit the implicit transaction block. The first command after a sync
+-- won't be seen as belonging to a pipeline.
+\startpipeline
+SET LOCAL statement_timeout='1h' \bind \g
+SHOW statement_timeout \bind \g
+\syncpipeline
+SHOW statement_timeout \bind \g
+SET LOCAL statement_timeout='2h' \bind \g
+SHOW statement_timeout \bind \g
+\endpipeline
+
+-- Reindex concurrently is forbidden in the middle of a pipeline
+\startpipeline
+SELECT $1 \bind 1 \g
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- Reindex concurrently will work if it's the first command of a pipeline
+\startpipeline
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- subtransactions are not allowed in pipeline mode
+\startpipeline
+SAVEPOINT a \bind \g
+SELECT $1 \bind 1 \g
+ROLLBACK TO SAVEPOINT a \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- Lock command will fail as first pipeline command is not seen as a transaction block
+\startpipeline
+LOCK psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- Lock command will succeed after the first command as pipeline will be seen as an implicit transaction block
+\startpipeline
+SELECT $1 \bind 1 \g
+LOCK psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- Vacuum command will work as the first command
+\startpipeline
+VACUUM psql_pipeline \bind \g
+\endpipeline
+
+-- Vacuum command will fail within pipeline implicit transaction
+\startpipeline
+SELECT 1 \bind \g
+VACUUM psql_pipeline \bind \g
+\endpipeline
+
+-- Vacuum command will work after a sync
+\startpipeline
+SELECT 1 \bind \g
+\syncpipeline
+VACUUM psql_pipeline \bind \g
+\endpipeline
--
2.39.5 (Apple Git-154)
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2024-12-11 23:53 Jelte Fennema-Nio <[email protected]>
parent: Anthonin Bonnefoy <[email protected]>
0 siblings, 1 reply; 41+ messages in thread
From: Jelte Fennema-Nio @ 2024-12-11 23:53 UTC (permalink / raw)
To: Anthonin Bonnefoy <[email protected]>; +Cc: Michael Paquier <[email protected]>; pgsql-hackers
On Tue, 10 Dec 2024 at 11:44, Anthonin Bonnefoy
<[email protected]> wrote:
> num_queries (2nd element in the pipeline status prompt) is now used to
> track queued queries that were not flushed (with a flush request or
> sync) to the server. It used to count both unflushed queries and
> flushed queries.
I skimmed the code a bit, but haven't looked closely at it yet. I did
try the patch out though. My thoughts below:
I think that new prompt is super useful, so useful in fact that I'd
suggest linking to it from the \startpipeline docs. I do think that
the wording in the docs could be a bit more precise:
1. The columns are not necessarily queries, they are messages or
commands. i.e. \parse and \bind_named both count as 1, even though
they form one query together.
2. messages not followed by \sync and \flushrequest, could very well
already "be sent to the server" (if the client buffer got full, or in
case of manual \flush). The main thing that \sync and \flushrequest do
is make sure that the server actually sends its own result back, even
if its buffer is not full yet.
The main feedback I have after playing around with this version is
that I'd like to have a \getresult (without the s), to only get a
single result. So that you can get results one by one, possibly
interleaved with some other queries again.
One thing I'm wondering is how useful the num_syncs count is in the
pipeline prompt, since you never really wait for a sync.
Regarding the usefulness of \flush. I agree it's not as useful as I
thought, because indeed \getresults already flushes everything. But
it's not completely useless either. The main way I was able to use it
interactively in a somewhat interesting way was to send it after a
long running query, and then while that's processing type up the next
query after it. Something like the following:
localhost jelte@postgres:5432-26274=
#> \startpipeline
Time: 0.000 ms
localhost jelte@postgres:5432-26274=
#|0,0,0|> select pg_sleep(5) \bind \g
Time: 0.000 ms
localhost jelte@postgres:5432-26274=
#|0,1,0|*> \flush
Time: 0.000 ms
localhost jelte@postgres:5432-26274=
#|0,1,0|*> select 1 \bind \g
Time: 0.000 ms
localhost jelte@postgres:5432-26274=
#|0,2,0|*> \syncpipeline
Time: 0.000 ms
localhost jelte@postgres:5432-26274=
#|1,0,2|*> \getresults
pg_sleep
──────────
(1 row)
?column?
──────────
1
(1 row)
Time: 0.348 ms
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2024-12-12 10:38 Anthonin Bonnefoy <[email protected]>
parent: Jelte Fennema-Nio <[email protected]>
0 siblings, 1 reply; 41+ messages in thread
From: Anthonin Bonnefoy @ 2024-12-12 10:38 UTC (permalink / raw)
To: Jelte Fennema-Nio <[email protected]>; +Cc: Michael Paquier <[email protected]>; pgsql-hackers
Thanks for the review!
On Thu, Dec 12, 2024 at 12:53 AM Jelte Fennema-Nio <[email protected]> wrote:
> I think that new prompt is super useful, so useful in fact that I'd
> suggest linking to it from the \startpipeline docs.
Good point, I've added a paragraph with the link to the %P prompt.
> I do think that
> the wording in the docs could be a bit more precise:
> 1. The columns are not necessarily queries, they are messages or
> commands. i.e. \parse and \bind_named both count as 1, even though
> they form one query together.
Yeah, I wasn't super happy with the num_queries and wording. I've
renamed the prompt columns to piped_syncs, piped_commands and
pending_results and added more precision on what counts as a command.
> 2. messages not followed by \sync and \flushrequest, could very well
> already "be sent to the server" (if the client buffer got full, or in
> case of manual \flush). The main thing that \sync and \flushrequest do
> is make sure that the server actually sends its own result back, even
> if its buffer is not full yet.
I had the wrong assumptions on what was happening. I've changed the
definition of pending_results to: "pending_results is the number of
commands that were followed by either a \flushrequest or a
\syncpipeline, forcing the server to send the results that can be
retrieved with \getresults."
> The main feedback I have after playing around with this version is
> that I'd like to have a \getresult (without the s), to only get a
> single result. So that you can get results one by one, possibly
> interleaved with some other queries again.
\getresults was easier to implement since it was more or less the
current behaviour :). I didn't want to add a new meta-command just for
that (I feel like I'm already adding a lot of them) so I've added a
num_results parameter to \getresults. You should be able to use
\getresults 1 to get a single result. A sync response count as a
result.
> One thing I'm wondering is how useful the num_syncs count is in the
> pipeline prompt, since you never really wait for a sync.
With the addition of the num_results parameter for \getresults,
knowing the number of syncs becomes more useful when results are
consumed one by one.
> Regarding the usefulness of \flush. I agree it's not as useful as I
> thought, because indeed \getresults already flushes everything. But
> it's not completely useless either. The main way I was able to use it
> interactively in a somewhat interesting way was to send it after a
> long running query, and then while that's processing type up the next
> query after it.
I feel there's a large overlap between \flush and \flushrequest. On
the other hand, if people want to test the behaviour of pushing data
with and without a flush request message, then \flush can be useful.
Attachments:
[application/octet-stream] v05-0001-Add-pipelining-support-in-psql.patch (54.0K, ../../CAO6_XqpLUSXcg9D=EZ-rJ97EMB0yqnpGuVKDk_8sy22kvFgNrQ@mail.gmail.com/2-v05-0001-Add-pipelining-support-in-psql.patch)
download | inline diff:
From 960707de76ac86887211059f335588b32c8a58f4 Mon Sep 17 00:00:00 2001
From: Anthonin Bonnefoy <[email protected]>
Date: Tue, 5 Nov 2024 10:26:54 +0100
Subject: Add pipelining support in psql
With \bind, \parse, \bind_named and \close, it is possible to issue
queries from psql using the extended protocol. However, it wasn't
possible to send those queries using pipelining and the only way to test
pipelined queries was through pgbench's tap tests.
This patch adds additional psql meta-commands to support pipelining:
\startpipeline, \endpipeline and \syncpipeline, mirroring the existing
meta-commands in pgbench. Additional meta-commands allow to flush and
read results of an ongoing pipeline: \flushrequest, \flush and \getresults
\startpipeline starts a new pipeline. All extended queries will be
queued until the end of the pipeline is reached.
\endpipeline ends an ongoing pipeline. All queued commands will be sent
to the server and all responses will be processed by the psql.
\syncpipeline queues a synchronisation point without flushing the
commands to the server
\flush Call PQflush on psql's connection
\flushrequest queues a flushrequest
\getresults reads server's results. Unsent data are automatically pushed
when \getresults is called
Those meta-commands will allow to test pipeline behaviour using
psql regression tests.
---
doc/src/sgml/ref/psql-ref.sgml | 96 +++++
src/bin/psql/command.c | 151 +++++++
src/bin/psql/common.c | 230 +++++++++-
src/bin/psql/help.c | 7 +
src/bin/psql/prompt.c | 12 +
src/bin/psql/settings.h | 15 +-
src/bin/psql/tab-complete.in.c | 8 +-
src/test/regress/expected/psql.out | 668 +++++++++++++++++++++++++++++
src/test/regress/sql/psql.sql | 399 +++++++++++++++++
9 files changed, 1576 insertions(+), 10 deletions(-)
diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml
index e42073ed748..db026d4e252 100644
--- a/doc/src/sgml/ref/psql-ref.sgml
+++ b/doc/src/sgml/ref/psql-ref.sgml
@@ -3562,6 +3562,82 @@ testdb=> <userinput>\setenv LESS -imx4F</userinput>
</listitem>
</varlistentry>
+ <varlistentry id="app-psql-meta-command-pipeline">
+ <term><literal>\startpipeline</literal></term>
+ <term><literal>\syncpipeline</literal></term>
+ <term><literal>\endpipeline</literal></term>
+ <term><literal>\flushrequest</literal></term>
+ <term><literal>\flush</literal></term>
+ <term><literal>\getresults [ <replaceable class="parameter">number_results</replaceable> ]</literal></term>
+
+ <listitem>
+ <para>
+ This group of commands implements pipelining of SQL statements.
+ A pipeline must begin with a <command>\startpipeline</command>
+ and end with an <command>\endpipeline</command>. In between there
+ may be any number of <command>\syncpipeline</command> commands,
+ which sends a <link linkend="protocol-flow-ext-query">sync message</link>
+ without ending the ongoing pipeline and flushing the send buffer.
+ In pipeline mode, statements are sent to the server without waiting
+ for the results of previous statements. See
+ <xref linkend="libpq-pipeline-mode"/> for more details.
+ </para>
+
+ <para>
+ Pipeline mode requires the use of extended query protocol. All queries need
+ to be sent using the meta-commands <literal>\bind</literal>,
+ <literal>\bind_named</literal>, <literal>\close</literal> or
+ <literal>\parse</literal>. While a pipeline is ongoing,
+ <literal>\g</literal> will append the current query buffer to the pipeline and
+ other meta-commands like <literal>\gx</literal> or <literal>\gdesc</literal>
+ are not allowed in pipeline mode.
+ </para>
+
+ <para>
+ <command>\flushrequest</command> appends a flush command to the pipeline,
+ allowing to read results with <command>\getresults</command> without issuing
+ a sync or ending the pipeline. <command>\getresults</command> will automatically
+ push unsent data to the server. <command>\flush</command> can be used to manually
+ push unsent data.
+ </para>
+
+ <para>
+ <command>\getresults</command> accepts an optional
+ <replaceable class="parameter">number_results</replaceable> parameter. If provided,
+ only the first <replaceable class="parameter">number_results</replaceable> pending
+ results will be read. If not provided or 0, all pending results are read. The
+ commands <literal>\bind</literal>, <literal>\bind_named</literal>,
+ <literal>\close</literal>, <literal>\parse</literal> and <literal>\syncpipeline</literal>
+ generate one result to get.
+ </para>
+
+ <para>
+ When pipeline mode is active, a dedicated prompt variable is
+ available to report the pipeline status. See
+ <xref linkend="app-psql-prompting-p-uc"/> for more details
+ </para>
+
+ <para>
+ Example:
+<programlisting>
+\startpipeline
+SELECT 1 \bind \g
+SELECT $1 \parse stmt1
+\bind_named stmt1 1 \g
+SELECT pg_current_xact_id() \bind \g
+\flushrequest
+\getresults
+\syncpipeline
+SELECT pg_current_xact_id() \bind \g
+\flushrequest
+\getresults 2
+\close stmt1
+\endpipeline
+</programlisting></para>
+
+ </listitem>
+ </varlistentry>
+
<varlistentry id="app-psql-meta-command-t-lc">
<term><literal>\t</literal></term>
@@ -4705,6 +4781,26 @@ testdb=> <userinput>INSERT INTO my_table VALUES (:'content');</userinput>
</listitem>
</varlistentry>
+ <varlistentry id="app-psql-prompting-p-uc">
+ <term><literal>%P</literal></term>
+ <listitem>
+ <para>
+ Pipeline status: an empty string when not in a pipeline,
+ or <literal>|piped_syncs,piped_commands,pending_results|</literal>
+ when in an ongoing pipeline, or <literal>+piped_syncs,piped_commands,pending_results+</literal>
+ when in an aborted pipeline. <literal>piped_syncs</literal> is the
+ number of syncs queued in the pipeline. <literal>piped_commands</literal>
+ is the number of commands generated by <literal>\bind</literal>,
+ <literal>\bind_named</literal>, <literal>\close</literal> or
+ <literal>\parse</literal> queued in the pipeline. <literal>pending_results</literal>
+ is the number of commands that were followed by either a
+ <command>\flushrequest</command> or a <command>\syncpipeline</command>,
+ forcing the server to send the results that can be retrieved with
+ <command>\getresults</command>.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry id="app-psql-prompting-r">
<term><literal>%R</literal></term>
<listitem>
diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c
index 1f3cbb11f7c..61d79b0257e 100644
--- a/src/bin/psql/command.c
+++ b/src/bin/psql/command.c
@@ -90,9 +90,12 @@ static backslashResult exec_command_else(PsqlScanState scan_state, ConditionalSt
PQExpBuffer query_buf);
static backslashResult exec_command_endif(PsqlScanState scan_state, ConditionalStack cstack,
PQExpBuffer query_buf);
+static backslashResult exec_command_endpipeline(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_encoding(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_errverbose(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_f(PsqlScanState scan_state, bool active_branch);
+static backslashResult exec_command_flush(PsqlScanState scan_state, bool active_branch);
+static backslashResult exec_command_flushrequest(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_g(PsqlScanState scan_state, bool active_branch,
const char *cmd);
static backslashResult process_command_g_options(char *first_option,
@@ -103,6 +106,7 @@ static backslashResult exec_command_gdesc(PsqlScanState scan_state, bool active_
static backslashResult exec_command_getenv(PsqlScanState scan_state, bool active_branch,
const char *cmd);
static backslashResult exec_command_gexec(PsqlScanState scan_state, bool active_branch);
+static backslashResult exec_command_getresults(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_gset(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_help(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_html(PsqlScanState scan_state, bool active_branch);
@@ -132,6 +136,8 @@ static backslashResult exec_command_setenv(PsqlScanState scan_state, bool active
const char *cmd);
static backslashResult exec_command_sf_sv(PsqlScanState scan_state, bool active_branch,
const char *cmd, bool is_func);
+static backslashResult exec_command_startpipeline(PsqlScanState scan_state, bool active_branch);
+static backslashResult exec_command_syncpipeline(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_t(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_T(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_timing(PsqlScanState scan_state, bool active_branch);
@@ -351,18 +357,26 @@ exec_command(const char *cmd,
status = exec_command_else(scan_state, cstack, query_buf);
else if (strcmp(cmd, "endif") == 0)
status = exec_command_endif(scan_state, cstack, query_buf);
+ else if (strcmp(cmd, "endpipeline") == 0)
+ status = exec_command_endpipeline(scan_state, active_branch);
else if (strcmp(cmd, "encoding") == 0)
status = exec_command_encoding(scan_state, active_branch);
else if (strcmp(cmd, "errverbose") == 0)
status = exec_command_errverbose(scan_state, active_branch);
else if (strcmp(cmd, "f") == 0)
status = exec_command_f(scan_state, active_branch);
+ else if (strcmp(cmd, "flush") == 0)
+ status = exec_command_flush(scan_state, active_branch);
+ else if (strcmp(cmd, "flushrequest") == 0)
+ status = exec_command_flushrequest(scan_state, active_branch);
else if (strcmp(cmd, "g") == 0 || strcmp(cmd, "gx") == 0)
status = exec_command_g(scan_state, active_branch, cmd);
else if (strcmp(cmd, "gdesc") == 0)
status = exec_command_gdesc(scan_state, active_branch);
else if (strcmp(cmd, "getenv") == 0)
status = exec_command_getenv(scan_state, active_branch, cmd);
+ else if (strcmp(cmd, "getresults") == 0)
+ status = exec_command_getresults(scan_state, active_branch);
else if (strcmp(cmd, "gexec") == 0)
status = exec_command_gexec(scan_state, active_branch);
else if (strcmp(cmd, "gset") == 0)
@@ -408,6 +422,10 @@ exec_command(const char *cmd,
status = exec_command_sf_sv(scan_state, active_branch, cmd, true);
else if (strcmp(cmd, "sv") == 0 || strcmp(cmd, "sv+") == 0)
status = exec_command_sf_sv(scan_state, active_branch, cmd, false);
+ else if (strcmp(cmd, "startpipeline") == 0)
+ status = exec_command_startpipeline(scan_state, active_branch);
+ else if (strcmp(cmd, "syncpipeline") == 0)
+ status = exec_command_syncpipeline(scan_state, active_branch);
else if (strcmp(cmd, "t") == 0)
status = exec_command_t(scan_state, active_branch);
else if (strcmp(cmd, "T") == 0)
@@ -1491,6 +1509,38 @@ exec_command_f(PsqlScanState scan_state, bool active_branch)
return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR;
}
+/*
+ * \flush -- call PQflush on the connection
+ */
+static backslashResult
+exec_command_flush(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_FLUSH;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
+/*
+ * \flushrequest -- send a flush request to the server
+ */
+static backslashResult
+exec_command_flushrequest(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_SEND_FLUSH_REQUEST;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
/*
* \g [(pset-option[=pset-value] ...)] [filename/shell-command]
* \gx [(pset-option[=pset-value] ...)] [filename/shell-command]
@@ -1526,6 +1576,13 @@ exec_command_g(PsqlScanState scan_state, bool active_branch, const char *cmd)
if (status == PSQL_CMD_SKIP_LINE && active_branch)
{
+ if (strcmp(cmd, "gx") == 0 && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pg_log_error("\\gx not allowed in pipeline mode");
+ clean_extended_state();
+ return PSQL_CMD_ERROR;
+ }
+
if (!fname)
pset.gfname = NULL;
else
@@ -1679,6 +1736,39 @@ exec_command_getenv(PsqlScanState scan_state, bool active_branch,
return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR;
}
+/*
+ * \getresults -- read results
+ */
+static backslashResult
+exec_command_getresults(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ char *opt;
+ int num_results;
+
+ pset.send_mode = PSQL_GET_RESULTS;
+ opt = psql_scan_slash_option(scan_state, OT_NORMAL, NULL, false);
+
+ pset.requested_results = 0;
+ if (opt != NULL)
+ {
+ num_results = atoi(opt);
+ if (num_results < 0)
+ {
+ pg_log_error("\\getresults: invalid number of requested results");
+ return PSQL_CMD_SKIP_LINE;
+ }
+ pset.requested_results = num_results;
+ }
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
+
/*
* \gexec -- send query and execute each field of result
*/
@@ -1689,6 +1779,12 @@ exec_command_gexec(PsqlScanState scan_state, bool active_branch)
if (active_branch)
{
+ if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pg_log_error("\\gexec not allowed in pipeline mode");
+ clean_extended_state();
+ return PSQL_CMD_ERROR;
+ }
pset.gexec_flag = true;
status = PSQL_CMD_SEND;
}
@@ -1709,6 +1805,13 @@ exec_command_gset(PsqlScanState scan_state, bool active_branch)
char *prefix = psql_scan_slash_option(scan_state,
OT_NORMAL, NULL, false);
+ if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pg_log_error("\\gset not allowed in pipeline mode");
+ clean_extended_state();
+ return PSQL_CMD_ERROR;
+ }
+
if (prefix)
pset.gset_prefix = prefix;
else
@@ -2672,6 +2775,54 @@ exec_command_sf_sv(PsqlScanState scan_state, bool active_branch,
return status;
}
+/*
+ * \startpipeline -- enter pipeline mode
+ */
+static backslashResult
+exec_command_startpipeline(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_START_PIPELINE_MODE;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
+/*
+ * \syncpipeline -- send a sync message to an active pipeline
+ */
+static backslashResult
+exec_command_syncpipeline(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_SEND_PIPELINE_SYNC;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
+/*
+ * \endpipeline -- end pipeline mode
+ */
+static backslashResult
+exec_command_endpipeline(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_END_PIPELINE_MODE;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
/*
* \t -- turn off table headers and row count
*/
diff --git a/src/bin/psql/common.c b/src/bin/psql/common.c
index 8a9211db41a..ebc9cd68ab0 100644
--- a/src/bin/psql/common.c
+++ b/src/bin/psql/common.c
@@ -415,10 +415,12 @@ AcceptResult(const PGresult *result, bool show_error)
case PGRES_EMPTY_QUERY:
case PGRES_COPY_IN:
case PGRES_COPY_OUT:
+ case PGRES_PIPELINE_SYNC:
/* Fine, do nothing */
OK = true;
break;
+ case PGRES_PIPELINE_ABORTED:
case PGRES_BAD_RESPONSE:
case PGRES_NONFATAL_ERROR:
case PGRES_FATAL_ERROR:
@@ -1050,6 +1052,7 @@ PrintQueryResult(PGresult *result, bool last,
success = true;
break;
+ case PGRES_PIPELINE_ABORTED:
case PGRES_BAD_RESPONSE:
case PGRES_NONFATAL_ERROR:
case PGRES_FATAL_ERROR:
@@ -1418,6 +1421,60 @@ DescribeQuery(const char *query, double *elapsed_msec)
return OK;
}
+/*
+ * Read and discard all results in an aborted pipeline.
+ *
+ * If a synchronisation point is found, we can stop discarding results as
+ * pipeline will switch back to an OK state. If no synchronisation point
+ * is available, we need to stop when there's no more pending results,
+ * otherwise, calling PQgetResults will block.
+ */
+static PGresult *
+discardAbortedPipelineResults(void)
+{
+ for (;;)
+ {
+ PGresult *res = PQgetResult(pset.db);
+ ExecStatusType result_status = PQresultStatus(res);
+
+ if (result_status == PGRES_PIPELINE_SYNC)
+ {
+ /*
+ * Found a synchronisation point. Decrementing the sync counter
+ * will be done by the caller
+ */
+ return res;
+ }
+ else if (res == NULL)
+ {
+ /* A query was processed, decrement the counters */
+ pset.pending_results--;
+ pset.requested_results--;
+ }
+ else if (res != NULL && result_status == PGRES_FATAL_ERROR)
+ return res;
+
+ if (pset.requested_results == 0)
+ /* We've read all requested results, exit */
+ return res;
+
+ if (pset.pending_results == 0 && pset.piped_syncs == 0)
+
+ /*
+ * There's no more results to get and there's no synchronisation
+ * point to stop at. This will leave the pipeline in an aborted
+ * state.
+ */
+ return res;
+
+ /*
+ * An aborted pipeline will have either NULL results or results in an
+ * PGRES_PIPELINE_ABORTED status
+ */
+ Assert(res == NULL || result_status == PGRES_PIPELINE_ABORTED);
+ PQclear(res);
+ }
+}
/*
* ExecQueryAndProcessResults: utility function for use by SendQuery()
@@ -1451,6 +1508,7 @@ ExecQueryAndProcessResults(const char *query,
bool timing = pset.timing;
bool success = false;
bool return_early = false;
+ bool end_pipeline = false;
instr_time before,
after;
PGresult *result;
@@ -1466,9 +1524,13 @@ ExecQueryAndProcessResults(const char *query,
{
case PSQL_SEND_EXTENDED_CLOSE:
success = PQsendClosePrepared(pset.db, pset.stmtName);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ pset.piped_commands++;
break;
case PSQL_SEND_EXTENDED_PARSE:
success = PQsendPrepare(pset.db, pset.stmtName, query, 0, NULL);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ pset.piped_commands++;
break;
case PSQL_SEND_EXTENDED_QUERY_PARAMS:
Assert(pset.stmtName == NULL);
@@ -1476,6 +1538,8 @@ ExecQueryAndProcessResults(const char *query,
pset.bind_nparams, NULL,
(const char *const *) pset.bind_params,
NULL, NULL, 0);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ pset.piped_commands++;
break;
case PSQL_SEND_EXTENDED_QUERY_PREPARED:
Assert(pset.stmtName != NULL);
@@ -1483,6 +1547,67 @@ ExecQueryAndProcessResults(const char *query,
pset.bind_nparams,
(const char *const *) pset.bind_params,
NULL, NULL, 0);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ pset.piped_commands++;
+ break;
+ case PSQL_START_PIPELINE_MODE:
+ success = PQenterPipelineMode(pset.db);
+ break;
+ case PSQL_END_PIPELINE_MODE:
+ success = PQpipelineSync(pset.db);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ /*
+ * End of the pipeline, all queued commands need to be
+ * processed
+ */
+ end_pipeline = true;
+ pset.piped_syncs++;
+ pset.pending_results += pset.piped_commands;
+ pset.piped_commands = 0;
+ pset.requested_results = pset.pending_results + pset.piped_syncs;
+ }
+ break;
+ case PSQL_SEND_PIPELINE_SYNC:
+ success = PQsendPipelineSync(pset.db);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pset.piped_syncs++;
+ pset.pending_results += pset.piped_commands;
+ pset.piped_commands = 0;
+ }
+ break;
+ case PSQL_FLUSH:
+ success = PQflush(pset.db);
+ break;
+ case PSQL_SEND_FLUSH_REQUEST:
+ success = PQsendFlushRequest(pset.db);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pset.pending_results += pset.piped_commands;
+ pset.piped_commands = 0;
+ }
+ break;
+ case PSQL_GET_RESULTS:
+ if (pset.pending_results == 0 && pset.piped_syncs == 0)
+ {
+ /*
+ * If no sync or flush request were sent, PQgetResult will
+ * block. Forbid the call to \getresults to avoid staying
+ * stuck
+ */
+ pg_log_info("No pending results to get");
+ success = false;
+ pset.requested_results = 0;
+ }
+ else
+ {
+ success = true;
+ /* Cap requested_results to the maximum known results */
+ if (pset.requested_results == 0 ||
+ pset.requested_results > (pset.pending_results + pset.piped_syncs))
+ pset.requested_results = pset.pending_results + pset.piped_syncs;
+ }
break;
case PSQL_SEND_QUERY:
success = PQsendQuery(pset.db, query);
@@ -1501,6 +1626,16 @@ ExecQueryAndProcessResults(const char *query,
return -1;
}
+ if (pset.requested_results == 0 && !end_pipeline &&
+ PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ /*
+ * We're in a pipeline and haven't reached the pipeline end or there
+ * was no request to read pipeline results, exit.
+ */
+ return 1;
+ }
+
/*
* Fetch the result in chunks if FETCH_COUNT is set, except when:
*
@@ -1548,7 +1683,7 @@ ExecQueryAndProcessResults(const char *query,
{
ExecStatusType result_status;
bool is_chunked_result = false;
- PGresult *next_result;
+ PGresult *next_result = NULL;
bool last;
if (!AcceptResult(result, false))
@@ -1571,6 +1706,9 @@ ExecQueryAndProcessResults(const char *query,
ClearOrSaveResult(result);
success = false;
+ if (result_status == PGRES_PIPELINE_ABORTED)
+ pg_log_info("Pipeline aborted, command didn't run");
+
/*
* switch to next result
*/
@@ -1585,6 +1723,15 @@ ExecQueryAndProcessResults(const char *query,
* ignore manually.
*/
result = NULL;
+ else if (end_pipeline || pset.requested_results > 0)
+
+ /*
+ * We have an error within a pipeline. All commands are
+ * aborted until the next synchronisation point. We need to
+ * consume all results until this synchronisation point, or
+ * stop when there's no more result to discard
+ */
+ result = discardAbortedPipelineResults();
else
result = PQgetResult(pset.db);
@@ -1771,12 +1918,66 @@ ExecQueryAndProcessResults(const char *query,
}
}
+ if (result_status == PGRES_PIPELINE_SYNC)
+ {
+ Assert(pset.piped_syncs > 0);
+
+ /*
+ * We have a sync response, decrease the sync and
+ * requested_results counters
+ */
+ pset.piped_syncs--;
+ pset.requested_results--;
+
+ /*
+ * After a synchronisation point, reset success state to print
+ * possible successful results that will be processed after this
+ */
+ success = true;
+
+ /*
+ * If all syncs were processed and pipeline end was requested,
+ * exit pipeline mode
+ */
+ if (end_pipeline && pset.piped_syncs == 0)
+ success &= PQexitPipelineMode(pset.db);
+ }
+ else if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF &&
+ result_status != PGRES_PIPELINE_SYNC)
+ {
+ /*
+ * We are in a pipeline and have a non sync response, decrease the
+ * results counters
+ */
+ pset.pending_results--;
+ pset.requested_results--;
+ }
+
/*
* Check PQgetResult() again. In the typical case of a single-command
* string, it will return NULL. Otherwise, we'll have other results
* to process. We need to do that to check whether this is the last.
*/
- next_result = PQgetResult(pset.db);
+ if (PQpipelineStatus(pset.db) == PQ_PIPELINE_OFF)
+ next_result = PQgetResult(pset.db);
+ else
+ {
+ /*
+ * In pipeline mode, a NULL result indicates the end of the
+ * current query being processed. Call PQgetResult to consume this
+ * NULL
+ */
+ if (result_status != PGRES_PIPELINE_SYNC)
+ {
+ next_result = PQgetResult(pset.db);
+ Assert(next_result == NULL);
+ }
+
+ /* We can now get the next result in the pipeline */
+ if (pset.requested_results > 0)
+ next_result = PQgetResult(pset.db);
+ }
+
last = (next_result == NULL);
/*
@@ -1798,8 +1999,12 @@ ExecQueryAndProcessResults(const char *query,
*elapsed_msec = INSTR_TIME_GET_MILLISEC(after);
}
- /* this may or may not print something depending on settings */
- if (result != NULL)
+ /*
+ * This may or may not print something depending on settings. A
+ * pipeline sync will have a non null result but doesn't have anything
+ * to print, thus ignore them
+ */
+ if (result != NULL && result_status != PGRES_PIPELINE_SYNC)
{
/*
* If results need to be printed into the file specified by \g,
@@ -1837,6 +2042,17 @@ ExecQueryAndProcessResults(const char *query,
/* close \g file if we opened it */
CloseGOutput(gfile_fout, gfile_is_pipe);
+ if (end_pipeline)
+ {
+ /* After a pipeline is processed, pipeline piped_syncs should be 0 */
+ Assert(pset.piped_syncs == 0);
+ /* all queries were processed */
+ Assert(pset.piped_commands == 0);
+ /* and all results were processed */
+ Assert(pset.pending_results == 0);
+ }
+ Assert(pset.requested_results == 0);
+
/* may need this to recover from conn loss during COPY */
if (!CheckConnection())
return -1;
@@ -2296,7 +2512,13 @@ clean_extended_state(void)
free(pset.stmtName);
pset.bind_params = NULL;
break;
+ case PSQL_START_PIPELINE_MODE: /* \startpipeline */
+ case PSQL_END_PIPELINE_MODE: /* \endpipeline */
+ case PSQL_SEND_PIPELINE_SYNC: /* \syncpipeline */
case PSQL_SEND_QUERY:
+ case PSQL_FLUSH: /* \flush */
+ case PSQL_GET_RESULTS: /* \getresults */
+ case PSQL_SEND_FLUSH_REQUEST: /* \flushrequest */
break;
}
diff --git a/src/bin/psql/help.c b/src/bin/psql/help.c
index 3f4afc2d141..c4dc218b9e7 100644
--- a/src/bin/psql/help.c
+++ b/src/bin/psql/help.c
@@ -167,15 +167,22 @@ slashUsage(unsigned short int pager)
HELP0(" \\close STMT_NAME close an existing prepared statement\n");
HELP0(" \\copyright show PostgreSQL usage and distribution terms\n");
HELP0(" \\crosstabview [COLUMNS] execute query and display result in crosstab\n");
+ HELP0(" \\endpipeline exit pipeline mode\n");
HELP0(" \\errverbose show most recent error message at maximum verbosity\n");
+ HELP0(" \\flush push unsent data to the server\n");
+ HELP0(" \\flushrequest send a flushrequest command\n");
HELP0(" \\g [(OPTIONS)] [FILE] execute query (and send result to file or |pipe);\n"
" \\g with no arguments is equivalent to a semicolon\n");
HELP0(" \\gdesc describe result of query, without executing it\n");
+ HELP0(" \\getresults [NUM_RES] read NUM_RES pending results. All pending results are\n"
+ " read if no argument is provided\n");
HELP0(" \\gexec execute query, then execute each value in its result\n");
HELP0(" \\gset [PREFIX] execute query and store result in psql variables\n");
HELP0(" \\gx [(OPTIONS)] [FILE] as \\g, but forces expanded output mode\n");
HELP0(" \\parse STMT_NAME create a prepared statement\n");
HELP0(" \\q quit psql\n");
+ HELP0(" \\startpipeline enter pipeline mode\n");
+ HELP0(" \\syncpipeline add a synchronisation point to an ongoing pipeline\n");
HELP0(" \\watch [[i=]SEC] [c=N] [m=MIN]\n"
" execute query every SEC seconds, up to N times,\n"
" stop if less than MIN rows are returned\n");
diff --git a/src/bin/psql/prompt.c b/src/bin/psql/prompt.c
index 0d99d00ac92..b1575105ade 100644
--- a/src/bin/psql/prompt.c
+++ b/src/bin/psql/prompt.c
@@ -31,6 +31,9 @@
* sockets, "[local:/dir/name]" if not default
* %m - like %M, but hostname only (before first dot), or always "[local]"
* %p - backend pid
+ * %P - pipeline status: no pipeline: empty
+ * ongoing pipeline: |piped_syncs,piped_commands,pending_results|
+ * aborted pipeline: +piped_syncs,piped_commands,pending_results+
* %> - database server port number
* %n - database user name
* %/ - current database
@@ -175,7 +178,16 @@ get_prompt(promptStatus_t status, ConditionalStack cstack)
snprintf(buf, sizeof(buf), "%d", pid);
}
break;
+ case 'P':
+ {
+ PGpipelineStatus status = PQpipelineStatus(pset.db);
+ if (status == PQ_PIPELINE_ON)
+ snprintf(buf, sizeof(buf), "|%d,%d,%d|", pset.piped_syncs, pset.piped_commands, pset.pending_results);
+ else if (status == PQ_PIPELINE_ABORTED)
+ snprintf(buf, sizeof(buf), "+%d,%d,%d+", pset.piped_syncs, pset.piped_commands, pset.pending_results);
+ break;
+ }
case '0':
case '1':
case '2':
diff --git a/src/bin/psql/settings.h b/src/bin/psql/settings.h
index a22de8ef78e..c1017b5ae44 100644
--- a/src/bin/psql/settings.h
+++ b/src/bin/psql/settings.h
@@ -23,8 +23,8 @@
#define DEFAULT_EDITOR_LINENUMBER_ARG "+"
#endif
-#define DEFAULT_PROMPT1 "%/%R%x%# "
-#define DEFAULT_PROMPT2 "%/%R%x%# "
+#define DEFAULT_PROMPT1 "%/%R%P%x%# "
+#define DEFAULT_PROMPT2 "%/%R%P%x%# "
#define DEFAULT_PROMPT3 ">> "
/*
@@ -69,6 +69,12 @@ typedef enum
PSQL_SEND_EXTENDED_PARSE,
PSQL_SEND_EXTENDED_QUERY_PARAMS,
PSQL_SEND_EXTENDED_QUERY_PREPARED,
+ PSQL_SEND_PIPELINE_SYNC,
+ PSQL_START_PIPELINE_MODE,
+ PSQL_END_PIPELINE_MODE,
+ PSQL_FLUSH,
+ PSQL_SEND_FLUSH_REQUEST,
+ PSQL_GET_RESULTS,
} PSQL_SEND_MODE;
typedef enum
@@ -111,6 +117,11 @@ typedef struct _psqlSettings
char **bind_params; /* parameters for extended query protocol call */
char *stmtName; /* prepared statement name used for extended
* query protocol commands */
+ int piped_commands; /* number of piped commands */
+ int piped_syncs; /* number of piped syncs */
+ int pending_results; /* number of results available to read */
+ int requested_results; /* number of requested results, include
+ * sync results */
bool crosstab_flag; /* one-shot request to crosstab result */
char *ctv_args[4]; /* \crosstabview arguments */
diff --git a/src/bin/psql/tab-complete.in.c b/src/bin/psql/tab-complete.in.c
index bbd08770c3d..de7240d8690 100644
--- a/src/bin/psql/tab-complete.in.c
+++ b/src/bin/psql/tab-complete.in.c
@@ -1867,9 +1867,9 @@ psql_completion(const char *text, int start, int end)
"\\drds", "\\drg", "\\dRs", "\\dRp", "\\ds",
"\\dt", "\\dT", "\\dv", "\\du", "\\dx", "\\dX", "\\dy",
"\\echo", "\\edit", "\\ef", "\\elif", "\\else", "\\encoding",
- "\\endif", "\\errverbose", "\\ev",
- "\\f",
- "\\g", "\\gdesc", "\\getenv", "\\gexec", "\\gset", "\\gx",
+ "\\endif", "\\endpipeline", "\\errverbose", "\\ev",
+ "\\f", "\\flush", "\\flushrequest",
+ "\\g", "\\gdesc", "\\getenv", "\\getresults", "\\gexec", "\\gset", "\\gx",
"\\help", "\\html",
"\\if", "\\include", "\\include_relative", "\\ir",
"\\list", "\\lo_import", "\\lo_export", "\\lo_list", "\\lo_unlink",
@@ -1877,7 +1877,7 @@ psql_completion(const char *text, int start, int end)
"\\parse", "\\password", "\\print", "\\prompt", "\\pset",
"\\qecho", "\\quit",
"\\reset",
- "\\s", "\\set", "\\setenv", "\\sf", "\\sv",
+ "\\s", "\\set", "\\setenv", "\\sf", "\\startpipeline", "\\sv", "\\syncpipeline",
"\\t", "\\T", "\\timing",
"\\unset",
"\\x",
diff --git a/src/test/regress/expected/psql.out b/src/test/regress/expected/psql.out
index 36dc31c16c4..ded510d50bc 100644
--- a/src/test/regress/expected/psql.out
+++ b/src/test/regress/expected/psql.out
@@ -6828,3 +6828,671 @@ CREATE TABLE defprivs (a int);
\pset null ''
DROP TABLE defprivs;
+-- pipelining
+CREATE TABLE psql_pipeline(a INTEGER PRIMARY KEY, s TEXT);
+-- single query
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+-- multiple queries
+\startpipeline
+SELECT $1 \bind 'val1' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val2 | val3
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val2 | val3
+(1 row)
+
+-- Test \flush
+\startpipeline
+\flush
+SELECT $1 \bind 'val1' \g
+\flush
+SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val2 | val3
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val2 | val3
+(1 row)
+
+-- send multiple syncs
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\syncpipeline
+\syncpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\syncpipeline
+SELECT $1, $2 \bind 'val4' 'val5' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val2 | val3
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val4 | val5
+(1 row)
+
+-- startpipeline shouldn't have any effect if already in a pipeline
+\startpipeline
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+-- Convert an implicit tx block to an explicit tx block
+\startpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 2 \g
+ROLLBACK \bind \g
+\endpipeline
+-- Multiple explicit transactions
+\startpipeline
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+ROLLBACK \bind \g
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+COMMIT \bind \g
+\endpipeline
+-- COPY FROM STDIN
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+-- COPY FROM STDIN with \flushrequest + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\flushrequest
+\getresults
+ ?column?
+----------
+ val1
+(1 row)
+
+message type 0x5a arrived from server while idle
+\endpipeline
+-- COPY FROM STDIN with \syncpipeline + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\syncpipeline
+\getresults
+ ?column?
+----------
+ val1
+(1 row)
+
+\endpipeline
+-- COPY TO STDOUT
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+1 \N
+2 test2
+3 test3
+4 test4
+-- COPY TO STDOUT with \flushrequest + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\flushrequest
+\getresults
+ ?column?
+----------
+ val1
+(1 row)
+
+1 \N
+2 test2
+3 test3
+4 test4
+\endpipeline
+-- COPY TO STDOUT with \syncpipeline + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\syncpipeline
+\getresults
+ ?column?
+----------
+ val1
+(1 row)
+
+1 \N
+2 test2
+3 test3
+4 test4
+\endpipeline
+-- Use \parse and \bind_named
+\startpipeline
+SELECT $1 \parse ''
+SELECT $1, $2 \parse ''
+SELECT $2 \parse pipeline_1
+\bind_named '' 1 2 \g
+\bind_named pipeline_1 2 \g
+\endpipeline
+ERROR: could not determine data type of parameter $1
+-- \getresults displays all results preceding a \flushrequest
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+\flushrequest
+\getresults
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+\endpipeline
+-- \getresults displays all results preceding a sync
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+\syncpipeline
+\getresults
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+\endpipeline
+-- \getresults immediately returns if there's no result to fetch
+\startpipeline
+\getresults
+No pending results to get
+SELECT $1 \bind 2 \g
+\getresults
+No pending results to get
+\flushrequest
+\endpipeline
+ ?column?
+----------
+ 2
+(1 row)
+
+\getresults
+No pending results to get
+-- \getresults only fetch results preceding a flushrequest
+\startpipeline
+SELECT $1 \bind 2 \g
+\flushrequest
+SELECT $1 \bind 2 \g
+\getresults
+ ?column?
+----------
+ 2
+(1 row)
+
+\endpipeline
+ ?column?
+----------
+ 2
+(1 row)
+
+-- \getresults only fetch results preceding a sync message
+\startpipeline
+SELECT $1 \bind 2 \g
+\syncpipeline
+SELECT $1 \bind 2 \g
+\getresults
+ ?column?
+----------
+ 2
+(1 row)
+
+\endpipeline
+ ?column?
+----------
+ 2
+(1 row)
+
+-- use pipeline with chunked results with both \getresults and \endpipeline
+\startpipeline
+\set FETCH_COUNT 10
+SELECT $1 \bind 2 \g
+\flushrequest
+\getresults
+ ?column?
+----------
+ 2
+(1 row)
+
+SELECT $1 \bind 2 \g
+\endpipeline
+ ?column?
+----------
+ 2
+(1 row)
+
+\unset FETCH_COUNT
+-- \getresults with specific number of requested results
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+SELECT $1 \bind 3 \g
+\syncpipeline
+\getresults 1
+ ?column?
+----------
+ 1
+(1 row)
+
+SELECT $1 \bind 4 \g
+\getresults 3
+ ?column?
+----------
+ 2
+(1 row)
+
+ ?column?
+----------
+ 3
+(1 row)
+
+\endpipeline
+ ?column?
+----------
+ 4
+(1 row)
+
+-- \syncpipeline count as one command to fetch for \getresults
+\startpipeline
+\syncpipeline
+\syncpipeline
+SELECT $1 \bind 1 \g
+\flushrequest
+\getresults 2
+\getresults 1
+ ?column?
+----------
+ 1
+(1 row)
+
+\endpipeline
+-- \getresults 0 should get all results
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+SELECT $1 \bind 3 \g
+\syncpipeline
+\getresults 0
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+ ?column?
+----------
+ 3
+(1 row)
+
+\endpipeline
+-- pipelining errors
+-- endpipeline outside of pipeline should fail
+\endpipeline
+cannot send pipeline when not in pipeline mode
+-- Query using simple protocol should not be sent and should leave the pipeline usable
+\startpipeline
+SELECT 1;
+PQsendQuery not allowed in pipeline mode
+SELECT $1 \bind 'val1' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+-- After an aborted pipeline, commands after a sync should be displayed
+\startpipeline
+SELECT $1 \bind \g
+\syncpipeline
+SELECT $1 \bind 1 \g
+\endpipeline
+ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
+ ?column?
+----------
+ 1
+(1 row)
+
+-- Incorrect number of parameters, the pipeline will be aborted and following queries won't be executed
+\startpipeline
+SELECT \bind 'val1' \g
+SELECT $1 \bind 'val1' \g
+\endpipeline
+ERROR: bind message supplies 1 parameters, but prepared statement "" requires 0
+-- An explicit transaction with an error needs to be rollbacked after the pipeline
+\startpipeline
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+ROLLBACK \bind \g
+\endpipeline
+ERROR: duplicate key value violates unique constraint "psql_pipeline_pkey"
+DETAIL: Key (a)=(1) already exists.
+ROLLBACK;
+-- \watch sends a simple query which won't be allowed within a pipeline
+\startpipeline
+SELECT \bind \g
+\watch 1
+PQsendQuery not allowed in pipeline mode
+
+\endpipeline
+--
+(1 row)
+
+-- \gdesc should fail as synchronous commands are not allowed in pipeline, pipeline should still be usable
+\startpipeline
+SELECT $1 \bind 1 \gdesc
+synchronous command execution functions are not allowed in pipeline mode
+SELECT $1 \bind 1 \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+-- \gset is not allowed, pipeline should still be usable
+\startpipeline
+SELECT $1 as i, $2 as j \parse ''
+SELECT $1 as k, $2 as l \parse 'second'
+\bind_named '' 1 2 \gset
+\gset not allowed in pipeline mode
+\bind_named second 1 2 \gset pref02_ \echo :pref02_i :pref02_j
+\gset not allowed in pipeline mode
+\bind_named '' 1 2 \g
+\endpipeline
+ i | j
+---+---
+ 1 | 2
+(1 row)
+
+-- \gx is not allowed, pipeline should still be usable
+\startpipeline
+SELECT $1 \bind 1 \gx
+\gx not allowed in pipeline mode
+\reset
+SELECT $1 \bind 1 \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+-- \gx warning should be emitted in an aborted pipeline
+\startpipeline
+SELECT $1 \bind \g
+\flushrequest
+\getresults
+ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
+SELECT $1 \bind 1 \gx
+\gx not allowed in pipeline mode
+\endpipeline
+-- \gexec is not allowed, pipeline should still be usable
+\startpipeline
+SELECT 'INSERT INTO psql_pipeline(a) SELECT generate_series(1, 10)' \parse 'insert_stmt'
+\bind_named insert_stmt \gexec
+\gexec not allowed in pipeline mode
+\bind_named insert_stmt \g
+SELECT COUNT(*) FROM psql_pipeline \bind \g
+\endpipeline
+ ?column?
+------------------------------------------------------------
+ INSERT INTO psql_pipeline(a) SELECT generate_series(1, 10)
+(1 row)
+
+ count
+-------
+ 4
+(1 row)
+
+-- After an error, pipeline is aborted and requires a sync to be reusable
+\startpipeline
+SELECT $1 \bind \g
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+\flushrequest
+\getresults
+ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
+-- Pipeline is aborted
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+-- Sync allows pipeline to recover
+\syncpipeline
+\getresults
+Pipeline aborted, command didn't run
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+\flushrequest
+\getresults
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 1
+(1 row)
+
+\endpipeline
+-- In an aborted pipeline, \getresults 1 aborted commands one at a time
+\startpipeline
+SELECT $1 \bind \g
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\syncpipeline
+\getresults 1
+ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
+\getresults 1
+Pipeline aborted, command didn't run
+\getresults 1
+Pipeline aborted, command didn't run
+\getresults 1
+Pipeline aborted, command didn't run
+\getresults 1
+\endpipeline
+-- Test chunked results with an aborted pipeline
+\startpipeline
+\set FETCH_COUNT 10
+SELECT $1 \bind \g
+\flushrequest
+\getresults
+ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
+SELECT $1 \bind \g
+\endpipeline
+fetching results in chunked mode failed
+Pipeline aborted, command didn't run
+\unset FETCH_COUNT
+-- \getresults returns an error when an incorrect number is provided
+\startpipeline
+\getresults -1
+\getresults: invalid number of requested results
+\endpipeline
+-- \getresults when there's no result shouldn't impact the following query
+\getresults 1
+No pending results to get
+select 1;
+ ?column?
+----------
+ 1
+(1 row)
+
+-- pipelining and transaction block behaviour
+-- set local will issue a warning when modifying a GUC outside of a transaction block
+-- The change will still be valid as a pipeline runs within an implicit transaction block
+-- Sending a sync will commit the implicit transaction block. The first command after a sync
+-- won't be seen as belonging to a pipeline.
+\startpipeline
+SET LOCAL statement_timeout='1h' \bind \g
+SHOW statement_timeout \bind \g
+\syncpipeline
+SHOW statement_timeout \bind \g
+SET LOCAL statement_timeout='2h' \bind \g
+SHOW statement_timeout \bind \g
+\endpipeline
+WARNING: SET LOCAL can only be used in transaction blocks
+ statement_timeout
+-------------------
+ 1h
+(1 row)
+
+ statement_timeout
+-------------------
+ 0
+(1 row)
+
+ statement_timeout
+-------------------
+ 2h
+(1 row)
+
+-- Reindex concurrently is forbidden in the middle of a pipeline
+\startpipeline
+SELECT $1 \bind 1 \g
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ERROR: REINDEX CONCURRENTLY cannot run inside a transaction block
+-- Reindex concurrently will work if it's the first command of a pipeline
+\startpipeline
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ ?column?
+----------
+ 2
+(1 row)
+
+-- subtransactions are not allowed in pipeline mode
+\startpipeline
+SAVEPOINT a \bind \g
+SELECT $1 \bind 1 \g
+ROLLBACK TO SAVEPOINT a \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ERROR: SAVEPOINT can only be used in transaction blocks
+-- Lock command will fail as first pipeline command is not seen as a transaction block
+\startpipeline
+LOCK psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ERROR: LOCK TABLE can only be used in transaction blocks
+-- Lock command will succeed after the first command as pipeline will be seen as an implicit transaction block
+\startpipeline
+SELECT $1 \bind 1 \g
+LOCK psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+-- Vacuum command will work as the first command
+\startpipeline
+VACUUM psql_pipeline \bind \g
+\endpipeline
+-- Vacuum command will fail within pipeline implicit transaction
+\startpipeline
+SELECT 1 \bind \g
+VACUUM psql_pipeline \bind \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ERROR: VACUUM cannot run inside a transaction block
+-- Vacuum command will work after a sync
+\startpipeline
+SELECT 1 \bind \g
+\syncpipeline
+VACUUM psql_pipeline \bind \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
diff --git a/src/test/regress/sql/psql.sql b/src/test/regress/sql/psql.sql
index c5021fc0b13..33b204edd7f 100644
--- a/src/test/regress/sql/psql.sql
+++ b/src/test/regress/sql/psql.sql
@@ -1929,3 +1929,402 @@ CREATE TABLE defprivs (a int);
\z defprivs
\pset null ''
DROP TABLE defprivs;
+
+-- pipelining
+CREATE TABLE psql_pipeline(a INTEGER PRIMARY KEY, s TEXT);
+
+-- single query
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\endpipeline
+
+-- multiple queries
+\startpipeline
+SELECT $1 \bind 'val1' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\endpipeline
+
+-- Test \flush
+\startpipeline
+\flush
+SELECT $1 \bind 'val1' \g
+\flush
+SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\endpipeline
+
+-- send multiple syncs
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\syncpipeline
+\syncpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\syncpipeline
+SELECT $1, $2 \bind 'val4' 'val5' \g
+\endpipeline
+
+-- startpipeline shouldn't have any effect if already in a pipeline
+\startpipeline
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\endpipeline
+
+-- Convert an implicit tx block to an explicit tx block
+\startpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 2 \g
+ROLLBACK \bind \g
+\endpipeline
+
+-- Multiple explicit transactions
+\startpipeline
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+ROLLBACK \bind \g
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+COMMIT \bind \g
+\endpipeline
+
+-- COPY FROM STDIN
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\endpipeline
+2 test2
+\.
+
+-- COPY FROM STDIN with \flushrequest + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\flushrequest
+\getresults
+3 test3
+\.
+\endpipeline
+
+-- COPY FROM STDIN with \syncpipeline + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\syncpipeline
+\getresults
+4 test4
+\.
+\endpipeline
+
+-- COPY TO STDOUT
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\endpipeline
+
+-- COPY TO STDOUT with \flushrequest + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\flushrequest
+\getresults
+\endpipeline
+
+-- COPY TO STDOUT with \syncpipeline + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\syncpipeline
+\getresults
+\endpipeline
+
+-- Use \parse and \bind_named
+\startpipeline
+SELECT $1 \parse ''
+SELECT $1, $2 \parse ''
+SELECT $2 \parse pipeline_1
+\bind_named '' 1 2 \g
+\bind_named pipeline_1 2 \g
+\endpipeline
+
+-- \getresults displays all results preceding a \flushrequest
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+\flushrequest
+\getresults
+\endpipeline
+
+-- \getresults displays all results preceding a sync
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+\syncpipeline
+\getresults
+\endpipeline
+
+-- \getresults immediately returns if there's no result to fetch
+\startpipeline
+\getresults
+SELECT $1 \bind 2 \g
+\getresults
+\flushrequest
+\endpipeline
+\getresults
+
+-- \getresults only fetch results preceding a flushrequest
+\startpipeline
+SELECT $1 \bind 2 \g
+\flushrequest
+SELECT $1 \bind 2 \g
+\getresults
+\endpipeline
+
+-- \getresults only fetch results preceding a sync message
+\startpipeline
+SELECT $1 \bind 2 \g
+\syncpipeline
+SELECT $1 \bind 2 \g
+\getresults
+\endpipeline
+
+-- use pipeline with chunked results with both \getresults and \endpipeline
+\startpipeline
+\set FETCH_COUNT 10
+SELECT $1 \bind 2 \g
+\flushrequest
+\getresults
+SELECT $1 \bind 2 \g
+\endpipeline
+\unset FETCH_COUNT
+
+-- \getresults with specific number of requested results
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+SELECT $1 \bind 3 \g
+\syncpipeline
+\getresults 1
+SELECT $1 \bind 4 \g
+\getresults 3
+\endpipeline
+
+-- \syncpipeline count as one command to fetch for \getresults
+\startpipeline
+\syncpipeline
+\syncpipeline
+SELECT $1 \bind 1 \g
+\flushrequest
+\getresults 2
+\getresults 1
+\endpipeline
+
+-- \getresults 0 should get all results
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+SELECT $1 \bind 3 \g
+\syncpipeline
+\getresults 0
+\endpipeline
+
+-- pipelining errors
+
+-- endpipeline outside of pipeline should fail
+\endpipeline
+
+-- Query using simple protocol should not be sent and should leave the pipeline usable
+\startpipeline
+SELECT 1;
+SELECT $1 \bind 'val1' \g
+\endpipeline
+
+-- After an aborted pipeline, commands after a sync should be displayed
+\startpipeline
+SELECT $1 \bind \g
+\syncpipeline
+SELECT $1 \bind 1 \g
+\endpipeline
+
+-- Incorrect number of parameters, the pipeline will be aborted and following queries won't be executed
+\startpipeline
+SELECT \bind 'val1' \g
+SELECT $1 \bind 'val1' \g
+\endpipeline
+
+-- An explicit transaction with an error needs to be rollbacked after the pipeline
+\startpipeline
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+ROLLBACK \bind \g
+\endpipeline
+ROLLBACK;
+
+-- \watch sends a simple query which won't be allowed within a pipeline
+\startpipeline
+SELECT \bind \g
+\watch 1
+\endpipeline
+
+-- \gdesc should fail as synchronous commands are not allowed in pipeline, pipeline should still be usable
+\startpipeline
+SELECT $1 \bind 1 \gdesc
+SELECT $1 \bind 1 \g
+\endpipeline
+
+-- \gset is not allowed, pipeline should still be usable
+\startpipeline
+SELECT $1 as i, $2 as j \parse ''
+SELECT $1 as k, $2 as l \parse 'second'
+\bind_named '' 1 2 \gset
+\bind_named second 1 2 \gset pref02_ \echo :pref02_i :pref02_j
+\bind_named '' 1 2 \g
+\endpipeline
+
+-- \gx is not allowed, pipeline should still be usable
+\startpipeline
+SELECT $1 \bind 1 \gx
+\reset
+SELECT $1 \bind 1 \g
+\endpipeline
+
+-- \gx warning should be emitted in an aborted pipeline
+\startpipeline
+SELECT $1 \bind \g
+\flushrequest
+\getresults
+SELECT $1 \bind 1 \gx
+\endpipeline
+
+-- \gexec is not allowed, pipeline should still be usable
+\startpipeline
+SELECT 'INSERT INTO psql_pipeline(a) SELECT generate_series(1, 10)' \parse 'insert_stmt'
+\bind_named insert_stmt \gexec
+\bind_named insert_stmt \g
+SELECT COUNT(*) FROM psql_pipeline \bind \g
+\endpipeline
+
+-- After an error, pipeline is aborted and requires a sync to be reusable
+\startpipeline
+SELECT $1 \bind \g
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+\flushrequest
+\getresults
+-- Pipeline is aborted
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+-- Sync allows pipeline to recover
+\syncpipeline
+\getresults
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+\flushrequest
+\getresults
+\endpipeline
+
+-- In an aborted pipeline, \getresults 1 aborted commands one at a time
+\startpipeline
+SELECT $1 \bind \g
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\syncpipeline
+\getresults 1
+\getresults 1
+\getresults 1
+\getresults 1
+\getresults 1
+\endpipeline
+
+-- Test chunked results with an aborted pipeline
+\startpipeline
+\set FETCH_COUNT 10
+SELECT $1 \bind \g
+\flushrequest
+\getresults
+SELECT $1 \bind \g
+\endpipeline
+\unset FETCH_COUNT
+
+-- \getresults returns an error when an incorrect number is provided
+\startpipeline
+\getresults -1
+\endpipeline
+
+-- \getresults when there's no result shouldn't impact the following query
+\getresults 1
+select 1;
+
+-- pipelining and transaction block behaviour
+
+-- set local will issue a warning when modifying a GUC outside of a transaction block
+-- The change will still be valid as a pipeline runs within an implicit transaction block
+-- Sending a sync will commit the implicit transaction block. The first command after a sync
+-- won't be seen as belonging to a pipeline.
+\startpipeline
+SET LOCAL statement_timeout='1h' \bind \g
+SHOW statement_timeout \bind \g
+\syncpipeline
+SHOW statement_timeout \bind \g
+SET LOCAL statement_timeout='2h' \bind \g
+SHOW statement_timeout \bind \g
+\endpipeline
+
+-- Reindex concurrently is forbidden in the middle of a pipeline
+\startpipeline
+SELECT $1 \bind 1 \g
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- Reindex concurrently will work if it's the first command of a pipeline
+\startpipeline
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- subtransactions are not allowed in pipeline mode
+\startpipeline
+SAVEPOINT a \bind \g
+SELECT $1 \bind 1 \g
+ROLLBACK TO SAVEPOINT a \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- Lock command will fail as first pipeline command is not seen as a transaction block
+\startpipeline
+LOCK psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- Lock command will succeed after the first command as pipeline will be seen as an implicit transaction block
+\startpipeline
+SELECT $1 \bind 1 \g
+LOCK psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- Vacuum command will work as the first command
+\startpipeline
+VACUUM psql_pipeline \bind \g
+\endpipeline
+
+-- Vacuum command will fail within pipeline implicit transaction
+\startpipeline
+SELECT 1 \bind \g
+VACUUM psql_pipeline \bind \g
+\endpipeline
+
+-- Vacuum command will work after a sync
+\startpipeline
+SELECT 1 \bind \g
+\syncpipeline
+VACUUM psql_pipeline \bind \g
+\endpipeline
--
2.39.5 (Apple Git-154)
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-01-10 10:59 Anthonin Bonnefoy <[email protected]>
parent: Anthonin Bonnefoy <[email protected]>
0 siblings, 1 reply; 41+ messages in thread
From: Anthonin Bonnefoy @ 2025-01-10 10:59 UTC (permalink / raw)
To: Jelte Fennema-Nio <[email protected]>; +Cc: Michael Paquier <[email protected]>; pgsql-hackers
> I feel there's a large overlap between \flush and \flushrequest. On
> the other hand, if people want to test the behaviour of pushing data
> with and without a flush request message, then \flush can be useful.
I've been looking into some issues related to a backend process stuck
in ClientWrite state blocking the WAL replay and it turns out that
pipelining + \flush was extremely useful to reproduce the issue. It
makes it possible to saturate the server->client socket buffer since
psql doesn't consume the results, easily triggering the state where
the process is stuck in ClientWrite. So I'm amending my position on
the usefulness of \flush and definitely see interesting usages.
I've also found out that I didn't correctly manage connection reset.
I've fixed this in v6 by resetting the pipeline counters on connection
reset and only calling discardAbortedPipelineResults if we're inside a
pipeline.
Attachments:
[application/octet-stream] v06-0001-Add-pipelining-support-in-psql.patch (54.7K, ../../CAO6_Xqq18VZji7G8MMEd8EiuuY1FrsZo=V1ch71aU7uZL6yqjg@mail.gmail.com/2-v06-0001-Add-pipelining-support-in-psql.patch)
download | inline diff:
From 08505130cb39dd8202d69561ab701f9a3d0c17f7 Mon Sep 17 00:00:00 2001
From: Anthonin Bonnefoy <[email protected]>
Date: Tue, 5 Nov 2024 10:26:54 +0100
Subject: Add pipelining support in psql
With \bind, \parse, \bind_named and \close, it is possible to issue
queries from psql using the extended protocol. However, it wasn't
possible to send those queries using pipelining and the only way to test
pipelined queries was through pgbench's tap tests.
This patch adds additional psql meta-commands to support pipelining:
\startpipeline, \endpipeline and \syncpipeline, mirroring the existing
meta-commands in pgbench. Additional meta-commands allow to flush and
read results of an ongoing pipeline: \flushrequest, \flush and \getresults
\startpipeline starts a new pipeline. All extended queries will be
queued until the end of the pipeline is reached.
\endpipeline ends an ongoing pipeline. All queued commands will be sent
to the server and all responses will be processed by the psql.
\syncpipeline queues a synchronisation point without flushing the
commands to the server
\flush Call PQflush on psql's connection
\flushrequest queues a flushrequest
\getresults reads server's results. Unsent data are automatically pushed
when \getresults is called
Those meta-commands will allow to test pipeline behaviour using
psql regression tests.
---
doc/src/sgml/ref/psql-ref.sgml | 96 +++++
src/bin/psql/command.c | 151 +++++++
src/bin/psql/common.c | 248 ++++++++++-
src/bin/psql/help.c | 7 +
src/bin/psql/prompt.c | 12 +
src/bin/psql/settings.h | 15 +-
src/bin/psql/tab-complete.in.c | 8 +-
src/test/regress/expected/psql.out | 668 +++++++++++++++++++++++++++++
src/test/regress/sql/psql.sql | 399 +++++++++++++++++
9 files changed, 1594 insertions(+), 10 deletions(-)
diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml
index 72f3347e53d..3bc77d2bb0e 100644
--- a/doc/src/sgml/ref/psql-ref.sgml
+++ b/doc/src/sgml/ref/psql-ref.sgml
@@ -3562,6 +3562,82 @@ testdb=> <userinput>\setenv LESS -imx4F</userinput>
</listitem>
</varlistentry>
+ <varlistentry id="app-psql-meta-command-pipeline">
+ <term><literal>\startpipeline</literal></term>
+ <term><literal>\syncpipeline</literal></term>
+ <term><literal>\endpipeline</literal></term>
+ <term><literal>\flushrequest</literal></term>
+ <term><literal>\flush</literal></term>
+ <term><literal>\getresults [ <replaceable class="parameter">number_results</replaceable> ]</literal></term>
+
+ <listitem>
+ <para>
+ This group of commands implements pipelining of SQL statements.
+ A pipeline must begin with a <command>\startpipeline</command>
+ and end with an <command>\endpipeline</command>. In between there
+ may be any number of <command>\syncpipeline</command> commands,
+ which sends a <link linkend="protocol-flow-ext-query">sync message</link>
+ without ending the ongoing pipeline and flushing the send buffer.
+ In pipeline mode, statements are sent to the server without waiting
+ for the results of previous statements. See
+ <xref linkend="libpq-pipeline-mode"/> for more details.
+ </para>
+
+ <para>
+ Pipeline mode requires the use of extended query protocol. All queries need
+ to be sent using the meta-commands <literal>\bind</literal>,
+ <literal>\bind_named</literal>, <literal>\close</literal> or
+ <literal>\parse</literal>. While a pipeline is ongoing,
+ <literal>\g</literal> will append the current query buffer to the pipeline and
+ other meta-commands like <literal>\gx</literal> or <literal>\gdesc</literal>
+ are not allowed in pipeline mode.
+ </para>
+
+ <para>
+ <command>\flushrequest</command> appends a flush command to the pipeline,
+ allowing to read results with <command>\getresults</command> without issuing
+ a sync or ending the pipeline. <command>\getresults</command> will automatically
+ push unsent data to the server. <command>\flush</command> can be used to manually
+ push unsent data.
+ </para>
+
+ <para>
+ <command>\getresults</command> accepts an optional
+ <replaceable class="parameter">number_results</replaceable> parameter. If provided,
+ only the first <replaceable class="parameter">number_results</replaceable> pending
+ results will be read. If not provided or 0, all pending results are read. The
+ commands <literal>\bind</literal>, <literal>\bind_named</literal>,
+ <literal>\close</literal>, <literal>\parse</literal> and <literal>\syncpipeline</literal>
+ generate one result to get.
+ </para>
+
+ <para>
+ When pipeline mode is active, a dedicated prompt variable is
+ available to report the pipeline status. See
+ <xref linkend="app-psql-prompting-p-uc"/> for more details
+ </para>
+
+ <para>
+ Example:
+<programlisting>
+\startpipeline
+SELECT 1 \bind \g
+SELECT $1 \parse stmt1
+\bind_named stmt1 1 \g
+SELECT pg_current_xact_id() \bind \g
+\flushrequest
+\getresults
+\syncpipeline
+SELECT pg_current_xact_id() \bind \g
+\flushrequest
+\getresults 2
+\close stmt1
+\endpipeline
+</programlisting></para>
+
+ </listitem>
+ </varlistentry>
+
<varlistentry id="app-psql-meta-command-t-lc">
<term><literal>\t</literal></term>
@@ -4719,6 +4795,26 @@ testdb=> <userinput>INSERT INTO my_table VALUES (:'content');</userinput>
</listitem>
</varlistentry>
+ <varlistentry id="app-psql-prompting-p-uc">
+ <term><literal>%P</literal></term>
+ <listitem>
+ <para>
+ Pipeline status: an empty string when not in a pipeline,
+ or <literal>|piped_syncs,piped_commands,pending_results|</literal>
+ when in an ongoing pipeline, or <literal>+piped_syncs,piped_commands,pending_results+</literal>
+ when in an aborted pipeline. <literal>piped_syncs</literal> is the
+ number of syncs queued in the pipeline. <literal>piped_commands</literal>
+ is the number of commands generated by <literal>\bind</literal>,
+ <literal>\bind_named</literal>, <literal>\close</literal> or
+ <literal>\parse</literal> queued in the pipeline. <literal>pending_results</literal>
+ is the number of commands that were followed by either a
+ <command>\flushrequest</command> or a <command>\syncpipeline</command>,
+ forcing the server to send the results that can be retrieved with
+ <command>\getresults</command>.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry id="app-psql-prompting-r">
<term><literal>%R</literal></term>
<listitem>
diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c
index 5dd4c2d2687..47682353fc6 100644
--- a/src/bin/psql/command.c
+++ b/src/bin/psql/command.c
@@ -90,9 +90,12 @@ static backslashResult exec_command_else(PsqlScanState scan_state, ConditionalSt
PQExpBuffer query_buf);
static backslashResult exec_command_endif(PsqlScanState scan_state, ConditionalStack cstack,
PQExpBuffer query_buf);
+static backslashResult exec_command_endpipeline(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_encoding(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_errverbose(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_f(PsqlScanState scan_state, bool active_branch);
+static backslashResult exec_command_flush(PsqlScanState scan_state, bool active_branch);
+static backslashResult exec_command_flushrequest(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_g(PsqlScanState scan_state, bool active_branch,
const char *cmd);
static backslashResult process_command_g_options(char *first_option,
@@ -103,6 +106,7 @@ static backslashResult exec_command_gdesc(PsqlScanState scan_state, bool active_
static backslashResult exec_command_getenv(PsqlScanState scan_state, bool active_branch,
const char *cmd);
static backslashResult exec_command_gexec(PsqlScanState scan_state, bool active_branch);
+static backslashResult exec_command_getresults(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_gset(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_help(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_html(PsqlScanState scan_state, bool active_branch);
@@ -132,6 +136,8 @@ static backslashResult exec_command_setenv(PsqlScanState scan_state, bool active
const char *cmd);
static backslashResult exec_command_sf_sv(PsqlScanState scan_state, bool active_branch,
const char *cmd, bool is_func);
+static backslashResult exec_command_startpipeline(PsqlScanState scan_state, bool active_branch);
+static backslashResult exec_command_syncpipeline(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_t(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_T(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_timing(PsqlScanState scan_state, bool active_branch);
@@ -351,18 +357,26 @@ exec_command(const char *cmd,
status = exec_command_else(scan_state, cstack, query_buf);
else if (strcmp(cmd, "endif") == 0)
status = exec_command_endif(scan_state, cstack, query_buf);
+ else if (strcmp(cmd, "endpipeline") == 0)
+ status = exec_command_endpipeline(scan_state, active_branch);
else if (strcmp(cmd, "encoding") == 0)
status = exec_command_encoding(scan_state, active_branch);
else if (strcmp(cmd, "errverbose") == 0)
status = exec_command_errverbose(scan_state, active_branch);
else if (strcmp(cmd, "f") == 0)
status = exec_command_f(scan_state, active_branch);
+ else if (strcmp(cmd, "flush") == 0)
+ status = exec_command_flush(scan_state, active_branch);
+ else if (strcmp(cmd, "flushrequest") == 0)
+ status = exec_command_flushrequest(scan_state, active_branch);
else if (strcmp(cmd, "g") == 0 || strcmp(cmd, "gx") == 0)
status = exec_command_g(scan_state, active_branch, cmd);
else if (strcmp(cmd, "gdesc") == 0)
status = exec_command_gdesc(scan_state, active_branch);
else if (strcmp(cmd, "getenv") == 0)
status = exec_command_getenv(scan_state, active_branch, cmd);
+ else if (strcmp(cmd, "getresults") == 0)
+ status = exec_command_getresults(scan_state, active_branch);
else if (strcmp(cmd, "gexec") == 0)
status = exec_command_gexec(scan_state, active_branch);
else if (strcmp(cmd, "gset") == 0)
@@ -408,6 +422,10 @@ exec_command(const char *cmd,
status = exec_command_sf_sv(scan_state, active_branch, cmd, true);
else if (strcmp(cmd, "sv") == 0 || strcmp(cmd, "sv+") == 0)
status = exec_command_sf_sv(scan_state, active_branch, cmd, false);
+ else if (strcmp(cmd, "startpipeline") == 0)
+ status = exec_command_startpipeline(scan_state, active_branch);
+ else if (strcmp(cmd, "syncpipeline") == 0)
+ status = exec_command_syncpipeline(scan_state, active_branch);
else if (strcmp(cmd, "t") == 0)
status = exec_command_t(scan_state, active_branch);
else if (strcmp(cmd, "T") == 0)
@@ -1491,6 +1509,38 @@ exec_command_f(PsqlScanState scan_state, bool active_branch)
return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR;
}
+/*
+ * \flush -- call PQflush on the connection
+ */
+static backslashResult
+exec_command_flush(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_FLUSH;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
+/*
+ * \flushrequest -- send a flush request to the server
+ */
+static backslashResult
+exec_command_flushrequest(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_SEND_FLUSH_REQUEST;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
/*
* \g [(pset-option[=pset-value] ...)] [filename/shell-command]
* \gx [(pset-option[=pset-value] ...)] [filename/shell-command]
@@ -1526,6 +1576,13 @@ exec_command_g(PsqlScanState scan_state, bool active_branch, const char *cmd)
if (status == PSQL_CMD_SKIP_LINE && active_branch)
{
+ if (strcmp(cmd, "gx") == 0 && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pg_log_error("\\gx not allowed in pipeline mode");
+ clean_extended_state();
+ return PSQL_CMD_ERROR;
+ }
+
if (!fname)
pset.gfname = NULL;
else
@@ -1679,6 +1736,39 @@ exec_command_getenv(PsqlScanState scan_state, bool active_branch,
return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR;
}
+/*
+ * \getresults -- read results
+ */
+static backslashResult
+exec_command_getresults(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ char *opt;
+ int num_results;
+
+ pset.send_mode = PSQL_GET_RESULTS;
+ opt = psql_scan_slash_option(scan_state, OT_NORMAL, NULL, false);
+
+ pset.requested_results = 0;
+ if (opt != NULL)
+ {
+ num_results = atoi(opt);
+ if (num_results < 0)
+ {
+ pg_log_error("\\getresults: invalid number of requested results");
+ return PSQL_CMD_SKIP_LINE;
+ }
+ pset.requested_results = num_results;
+ }
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
+
/*
* \gexec -- send query and execute each field of result
*/
@@ -1689,6 +1779,12 @@ exec_command_gexec(PsqlScanState scan_state, bool active_branch)
if (active_branch)
{
+ if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pg_log_error("\\gexec not allowed in pipeline mode");
+ clean_extended_state();
+ return PSQL_CMD_ERROR;
+ }
pset.gexec_flag = true;
status = PSQL_CMD_SEND;
}
@@ -1709,6 +1805,13 @@ exec_command_gset(PsqlScanState scan_state, bool active_branch)
char *prefix = psql_scan_slash_option(scan_state,
OT_NORMAL, NULL, false);
+ if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pg_log_error("\\gset not allowed in pipeline mode");
+ clean_extended_state();
+ return PSQL_CMD_ERROR;
+ }
+
if (prefix)
pset.gset_prefix = prefix;
else
@@ -2672,6 +2775,54 @@ exec_command_sf_sv(PsqlScanState scan_state, bool active_branch,
return status;
}
+/*
+ * \startpipeline -- enter pipeline mode
+ */
+static backslashResult
+exec_command_startpipeline(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_START_PIPELINE_MODE;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
+/*
+ * \syncpipeline -- send a sync message to an active pipeline
+ */
+static backslashResult
+exec_command_syncpipeline(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_SEND_PIPELINE_SYNC;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
+/*
+ * \endpipeline -- end pipeline mode
+ */
+static backslashResult
+exec_command_endpipeline(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_END_PIPELINE_MODE;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
/*
* \t -- turn off table headers and row count
*/
diff --git a/src/bin/psql/common.c b/src/bin/psql/common.c
index f1a5291c13b..da39bff183d 100644
--- a/src/bin/psql/common.c
+++ b/src/bin/psql/common.c
@@ -121,6 +121,18 @@ CloseGOutput(FILE *gfile_fout, bool is_pipe)
}
}
+/*
+ * Reset pset pipeline state
+ */
+static void
+pipelineReset(void)
+{
+ pset.piped_syncs = 0;
+ pset.piped_commands = 0;
+ pset.pending_results = 0;
+ pset.requested_results = 0;
+}
+
/*
* setQFout
* -- handler for -o command line option and \o command
@@ -354,6 +366,7 @@ CheckConnection(void)
fprintf(stderr, _("The connection to the server was lost. Attempting reset: "));
PQreset(pset.db);
+ pipelineReset();
OK = ConnectionUp();
if (!OK)
{
@@ -415,10 +428,12 @@ AcceptResult(const PGresult *result, bool show_error)
case PGRES_EMPTY_QUERY:
case PGRES_COPY_IN:
case PGRES_COPY_OUT:
+ case PGRES_PIPELINE_SYNC:
/* Fine, do nothing */
OK = true;
break;
+ case PGRES_PIPELINE_ABORTED:
case PGRES_BAD_RESPONSE:
case PGRES_NONFATAL_ERROR:
case PGRES_FATAL_ERROR:
@@ -1050,6 +1065,7 @@ PrintQueryResult(PGresult *result, bool last,
success = true;
break;
+ case PGRES_PIPELINE_ABORTED:
case PGRES_BAD_RESPONSE:
case PGRES_NONFATAL_ERROR:
case PGRES_FATAL_ERROR:
@@ -1418,6 +1434,60 @@ DescribeQuery(const char *query, double *elapsed_msec)
return OK;
}
+/*
+ * Read and discard all results in an aborted pipeline.
+ *
+ * If a synchronisation point is found, we can stop discarding results as
+ * pipeline will switch back to an OK state. If no synchronisation point
+ * is available, we need to stop when there's no more pending results,
+ * otherwise, calling PQgetResults will block.
+ */
+static PGresult *
+discardAbortedPipelineResults(void)
+{
+ for (;;)
+ {
+ PGresult *res = PQgetResult(pset.db);
+ ExecStatusType result_status = PQresultStatus(res);
+
+ if (result_status == PGRES_PIPELINE_SYNC)
+ {
+ /*
+ * Found a synchronisation point. Decrementing the sync counter
+ * will be done by the caller
+ */
+ return res;
+ }
+ else if (res == NULL)
+ {
+ /* A query was processed, decrement the counters */
+ pset.pending_results--;
+ pset.requested_results--;
+ }
+ else if (res != NULL && result_status == PGRES_FATAL_ERROR)
+ return res;
+
+ if (pset.requested_results == 0)
+ /* We've read all requested results, exit */
+ return res;
+
+ if (pset.pending_results == 0 && pset.piped_syncs == 0)
+
+ /*
+ * There's no more results to get and there's no synchronisation
+ * point to stop at. This will leave the pipeline in an aborted
+ * state.
+ */
+ return res;
+
+ /*
+ * An aborted pipeline will have either NULL results or results in an
+ * PGRES_PIPELINE_ABORTED status
+ */
+ Assert(res == NULL || result_status == PGRES_PIPELINE_ABORTED);
+ PQclear(res);
+ }
+}
/*
* ExecQueryAndProcessResults: utility function for use by SendQuery()
@@ -1451,6 +1521,7 @@ ExecQueryAndProcessResults(const char *query,
bool timing = pset.timing;
bool success = false;
bool return_early = false;
+ bool end_pipeline = false;
instr_time before,
after;
PGresult *result;
@@ -1466,9 +1537,13 @@ ExecQueryAndProcessResults(const char *query,
{
case PSQL_SEND_EXTENDED_CLOSE:
success = PQsendClosePrepared(pset.db, pset.stmtName);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ pset.piped_commands++;
break;
case PSQL_SEND_EXTENDED_PARSE:
success = PQsendPrepare(pset.db, pset.stmtName, query, 0, NULL);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ pset.piped_commands++;
break;
case PSQL_SEND_EXTENDED_QUERY_PARAMS:
Assert(pset.stmtName == NULL);
@@ -1476,6 +1551,8 @@ ExecQueryAndProcessResults(const char *query,
pset.bind_nparams, NULL,
(const char *const *) pset.bind_params,
NULL, NULL, 0);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ pset.piped_commands++;
break;
case PSQL_SEND_EXTENDED_QUERY_PREPARED:
Assert(pset.stmtName != NULL);
@@ -1483,6 +1560,67 @@ ExecQueryAndProcessResults(const char *query,
pset.bind_nparams,
(const char *const *) pset.bind_params,
NULL, NULL, 0);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ pset.piped_commands++;
+ break;
+ case PSQL_START_PIPELINE_MODE:
+ success = PQenterPipelineMode(pset.db);
+ break;
+ case PSQL_END_PIPELINE_MODE:
+ success = PQpipelineSync(pset.db);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ /*
+ * End of the pipeline, all queued commands need to be
+ * processed
+ */
+ end_pipeline = true;
+ pset.piped_syncs++;
+ pset.pending_results += pset.piped_commands;
+ pset.piped_commands = 0;
+ pset.requested_results = pset.pending_results + pset.piped_syncs;
+ }
+ break;
+ case PSQL_SEND_PIPELINE_SYNC:
+ success = PQsendPipelineSync(pset.db);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pset.piped_syncs++;
+ pset.pending_results += pset.piped_commands;
+ pset.piped_commands = 0;
+ }
+ break;
+ case PSQL_FLUSH:
+ success = PQflush(pset.db);
+ break;
+ case PSQL_SEND_FLUSH_REQUEST:
+ success = PQsendFlushRequest(pset.db);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pset.pending_results += pset.piped_commands;
+ pset.piped_commands = 0;
+ }
+ break;
+ case PSQL_GET_RESULTS:
+ if (pset.pending_results == 0 && pset.piped_syncs == 0)
+ {
+ /*
+ * If no sync or flush request were sent, PQgetResult will
+ * block. Forbid the call to \getresults to avoid staying
+ * stuck
+ */
+ pg_log_info("No pending results to get");
+ success = false;
+ pset.requested_results = 0;
+ }
+ else
+ {
+ success = true;
+ /* Cap requested_results to the maximum known results */
+ if (pset.requested_results == 0 ||
+ pset.requested_results > (pset.pending_results + pset.piped_syncs))
+ pset.requested_results = pset.pending_results + pset.piped_syncs;
+ }
break;
case PSQL_SEND_QUERY:
success = PQsendQuery(pset.db, query);
@@ -1501,6 +1639,16 @@ ExecQueryAndProcessResults(const char *query,
return -1;
}
+ if (pset.requested_results == 0 && !end_pipeline &&
+ PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ /*
+ * We're in a pipeline and haven't reached the pipeline end or there
+ * was no request to read pipeline results, exit.
+ */
+ return 1;
+ }
+
/*
* Fetch the result in chunks if FETCH_COUNT is set, except when:
*
@@ -1548,7 +1696,7 @@ ExecQueryAndProcessResults(const char *query,
{
ExecStatusType result_status;
bool is_chunked_result = false;
- PGresult *next_result;
+ PGresult *next_result = NULL;
bool last;
if (!AcceptResult(result, false))
@@ -1571,6 +1719,9 @@ ExecQueryAndProcessResults(const char *query,
ClearOrSaveResult(result);
success = false;
+ if (result_status == PGRES_PIPELINE_ABORTED)
+ pg_log_info("Pipeline aborted, command didn't run");
+
/*
* switch to next result
*/
@@ -1585,6 +1736,20 @@ ExecQueryAndProcessResults(const char *query,
* ignore manually.
*/
result = NULL;
+ else if ((end_pipeline || pset.requested_results > 0)
+ && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+
+ /*
+ * We have an error within a pipeline. All commands are
+ * aborted until the next synchronisation point. We need to
+ * consume all results until this synchronisation point, or
+ * stop when there's no more result to discard
+ *
+ * Checking pipeline status is necessary in case the
+ * connection was reset. The new connection isn't in any kind
+ * of pipeline state and thus has no result to discard
+ */
+ result = discardAbortedPipelineResults();
else
result = PQgetResult(pset.db);
@@ -1771,12 +1936,66 @@ ExecQueryAndProcessResults(const char *query,
}
}
+ if (result_status == PGRES_PIPELINE_SYNC)
+ {
+ Assert(pset.piped_syncs > 0);
+
+ /*
+ * We have a sync response, decrease the sync and
+ * requested_results counters
+ */
+ pset.piped_syncs--;
+ pset.requested_results--;
+
+ /*
+ * After a synchronisation point, reset success state to print
+ * possible successful results that will be processed after this
+ */
+ success = true;
+
+ /*
+ * If all syncs were processed and pipeline end was requested,
+ * exit pipeline mode
+ */
+ if (end_pipeline && pset.piped_syncs == 0)
+ success &= PQexitPipelineMode(pset.db);
+ }
+ else if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF &&
+ result_status != PGRES_PIPELINE_SYNC)
+ {
+ /*
+ * We are in a pipeline and have a non sync response, decrease the
+ * results counters
+ */
+ pset.pending_results--;
+ pset.requested_results--;
+ }
+
/*
* Check PQgetResult() again. In the typical case of a single-command
* string, it will return NULL. Otherwise, we'll have other results
* to process. We need to do that to check whether this is the last.
*/
- next_result = PQgetResult(pset.db);
+ if (PQpipelineStatus(pset.db) == PQ_PIPELINE_OFF)
+ next_result = PQgetResult(pset.db);
+ else
+ {
+ /*
+ * In pipeline mode, a NULL result indicates the end of the
+ * current query being processed. Call PQgetResult to consume this
+ * NULL
+ */
+ if (result_status != PGRES_PIPELINE_SYNC)
+ {
+ next_result = PQgetResult(pset.db);
+ Assert(next_result == NULL);
+ }
+
+ /* We can now get the next result in the pipeline */
+ if (pset.requested_results > 0)
+ next_result = PQgetResult(pset.db);
+ }
+
last = (next_result == NULL);
/*
@@ -1798,8 +2017,12 @@ ExecQueryAndProcessResults(const char *query,
*elapsed_msec = INSTR_TIME_GET_MILLISEC(after);
}
- /* this may or may not print something depending on settings */
- if (result != NULL)
+ /*
+ * This may or may not print something depending on settings. A
+ * pipeline sync will have a non null result but doesn't have anything
+ * to print, thus ignore them
+ */
+ if (result != NULL && result_status != PGRES_PIPELINE_SYNC)
{
/*
* If results need to be printed into the file specified by \g,
@@ -1837,6 +2060,17 @@ ExecQueryAndProcessResults(const char *query,
/* close \g file if we opened it */
CloseGOutput(gfile_fout, gfile_is_pipe);
+ if (end_pipeline)
+ {
+ /* After a pipeline is processed, pipeline piped_syncs should be 0 */
+ Assert(pset.piped_syncs == 0);
+ /* all queries were processed */
+ Assert(pset.piped_commands == 0);
+ /* and all results were processed */
+ Assert(pset.pending_results == 0);
+ }
+ Assert(pset.requested_results == 0);
+
/* may need this to recover from conn loss during COPY */
if (!CheckConnection())
return -1;
@@ -2296,7 +2530,13 @@ clean_extended_state(void)
free(pset.stmtName);
pset.bind_params = NULL;
break;
+ case PSQL_START_PIPELINE_MODE: /* \startpipeline */
+ case PSQL_END_PIPELINE_MODE: /* \endpipeline */
+ case PSQL_SEND_PIPELINE_SYNC: /* \syncpipeline */
case PSQL_SEND_QUERY:
+ case PSQL_FLUSH: /* \flush */
+ case PSQL_GET_RESULTS: /* \getresults */
+ case PSQL_SEND_FLUSH_REQUEST: /* \flushrequest */
break;
}
diff --git a/src/bin/psql/help.c b/src/bin/psql/help.c
index fda83465efa..92df53ffd67 100644
--- a/src/bin/psql/help.c
+++ b/src/bin/psql/help.c
@@ -167,15 +167,22 @@ slashUsage(unsigned short int pager)
HELP0(" \\close STMT_NAME close an existing prepared statement\n");
HELP0(" \\copyright show PostgreSQL usage and distribution terms\n");
HELP0(" \\crosstabview [COLUMNS] execute query and display result in crosstab\n");
+ HELP0(" \\endpipeline exit pipeline mode\n");
HELP0(" \\errverbose show most recent error message at maximum verbosity\n");
+ HELP0(" \\flush push unsent data to the server\n");
+ HELP0(" \\flushrequest send a flushrequest command\n");
HELP0(" \\g [(OPTIONS)] [FILE] execute query (and send result to file or |pipe);\n"
" \\g with no arguments is equivalent to a semicolon\n");
HELP0(" \\gdesc describe result of query, without executing it\n");
+ HELP0(" \\getresults [NUM_RES] read NUM_RES pending results. All pending results are\n"
+ " read if no argument is provided\n");
HELP0(" \\gexec execute query, then execute each value in its result\n");
HELP0(" \\gset [PREFIX] execute query and store result in psql variables\n");
HELP0(" \\gx [(OPTIONS)] [FILE] as \\g, but forces expanded output mode\n");
HELP0(" \\parse STMT_NAME create a prepared statement\n");
HELP0(" \\q quit psql\n");
+ HELP0(" \\startpipeline enter pipeline mode\n");
+ HELP0(" \\syncpipeline add a synchronisation point to an ongoing pipeline\n");
HELP0(" \\watch [[i=]SEC] [c=N] [m=MIN]\n"
" execute query every SEC seconds, up to N times,\n"
" stop if less than MIN rows are returned\n");
diff --git a/src/bin/psql/prompt.c b/src/bin/psql/prompt.c
index 08a14feb3c3..a56ea0787c1 100644
--- a/src/bin/psql/prompt.c
+++ b/src/bin/psql/prompt.c
@@ -31,6 +31,9 @@
* sockets, "[local:/dir/name]" if not default
* %m - like %M, but hostname only (before first dot), or always "[local]"
* %p - backend pid
+ * %P - pipeline status: no pipeline: empty
+ * ongoing pipeline: |piped_syncs,piped_commands,pending_results|
+ * aborted pipeline: +piped_syncs,piped_commands,pending_results+
* %> - database server port number
* %n - database user name
* %s - service
@@ -181,7 +184,16 @@ get_prompt(promptStatus_t status, ConditionalStack cstack)
snprintf(buf, sizeof(buf), "%d", pid);
}
break;
+ case 'P':
+ {
+ PGpipelineStatus status = PQpipelineStatus(pset.db);
+ if (status == PQ_PIPELINE_ON)
+ snprintf(buf, sizeof(buf), "|%d,%d,%d|", pset.piped_syncs, pset.piped_commands, pset.pending_results);
+ else if (status == PQ_PIPELINE_ABORTED)
+ snprintf(buf, sizeof(buf), "+%d,%d,%d+", pset.piped_syncs, pset.piped_commands, pset.pending_results);
+ break;
+ }
case '0':
case '1':
case '2':
diff --git a/src/bin/psql/settings.h b/src/bin/psql/settings.h
index 2a8fe12eb55..4c70d21c409 100644
--- a/src/bin/psql/settings.h
+++ b/src/bin/psql/settings.h
@@ -23,8 +23,8 @@
#define DEFAULT_EDITOR_LINENUMBER_ARG "+"
#endif
-#define DEFAULT_PROMPT1 "%/%R%x%# "
-#define DEFAULT_PROMPT2 "%/%R%x%# "
+#define DEFAULT_PROMPT1 "%/%R%P%x%# "
+#define DEFAULT_PROMPT2 "%/%R%P%x%# "
#define DEFAULT_PROMPT3 ">> "
/*
@@ -69,6 +69,12 @@ typedef enum
PSQL_SEND_EXTENDED_PARSE,
PSQL_SEND_EXTENDED_QUERY_PARAMS,
PSQL_SEND_EXTENDED_QUERY_PREPARED,
+ PSQL_SEND_PIPELINE_SYNC,
+ PSQL_START_PIPELINE_MODE,
+ PSQL_END_PIPELINE_MODE,
+ PSQL_FLUSH,
+ PSQL_SEND_FLUSH_REQUEST,
+ PSQL_GET_RESULTS,
} PSQL_SEND_MODE;
typedef enum
@@ -111,6 +117,11 @@ typedef struct _psqlSettings
char **bind_params; /* parameters for extended query protocol call */
char *stmtName; /* prepared statement name used for extended
* query protocol commands */
+ int piped_commands; /* number of piped commands */
+ int piped_syncs; /* number of piped syncs */
+ int pending_results; /* number of results available to read */
+ int requested_results; /* number of requested results, include
+ * sync results */
bool crosstab_flag; /* one-shot request to crosstab result */
char *ctv_args[4]; /* \crosstabview arguments */
diff --git a/src/bin/psql/tab-complete.in.c b/src/bin/psql/tab-complete.in.c
index 81cbf10aa28..bd3a78d909d 100644
--- a/src/bin/psql/tab-complete.in.c
+++ b/src/bin/psql/tab-complete.in.c
@@ -1867,9 +1867,9 @@ psql_completion(const char *text, int start, int end)
"\\drds", "\\drg", "\\dRs", "\\dRp", "\\ds",
"\\dt", "\\dT", "\\dv", "\\du", "\\dx", "\\dX", "\\dy",
"\\echo", "\\edit", "\\ef", "\\elif", "\\else", "\\encoding",
- "\\endif", "\\errverbose", "\\ev",
- "\\f",
- "\\g", "\\gdesc", "\\getenv", "\\gexec", "\\gset", "\\gx",
+ "\\endif", "\\endpipeline", "\\errverbose", "\\ev",
+ "\\f", "\\flush", "\\flushrequest",
+ "\\g", "\\gdesc", "\\getenv", "\\getresults", "\\gexec", "\\gset", "\\gx",
"\\help", "\\html",
"\\if", "\\include", "\\include_relative", "\\ir",
"\\list", "\\lo_import", "\\lo_export", "\\lo_list", "\\lo_unlink",
@@ -1877,7 +1877,7 @@ psql_completion(const char *text, int start, int end)
"\\parse", "\\password", "\\print", "\\prompt", "\\pset",
"\\qecho", "\\quit",
"\\reset",
- "\\s", "\\set", "\\setenv", "\\sf", "\\sv",
+ "\\s", "\\set", "\\setenv", "\\sf", "\\startpipeline", "\\sv", "\\syncpipeline",
"\\t", "\\T", "\\timing",
"\\unset",
"\\x",
diff --git a/src/test/regress/expected/psql.out b/src/test/regress/expected/psql.out
index 36dc31c16c4..ded510d50bc 100644
--- a/src/test/regress/expected/psql.out
+++ b/src/test/regress/expected/psql.out
@@ -6828,3 +6828,671 @@ CREATE TABLE defprivs (a int);
\pset null ''
DROP TABLE defprivs;
+-- pipelining
+CREATE TABLE psql_pipeline(a INTEGER PRIMARY KEY, s TEXT);
+-- single query
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+-- multiple queries
+\startpipeline
+SELECT $1 \bind 'val1' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val2 | val3
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val2 | val3
+(1 row)
+
+-- Test \flush
+\startpipeline
+\flush
+SELECT $1 \bind 'val1' \g
+\flush
+SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val2 | val3
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val2 | val3
+(1 row)
+
+-- send multiple syncs
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\syncpipeline
+\syncpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\syncpipeline
+SELECT $1, $2 \bind 'val4' 'val5' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val2 | val3
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val4 | val5
+(1 row)
+
+-- startpipeline shouldn't have any effect if already in a pipeline
+\startpipeline
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+-- Convert an implicit tx block to an explicit tx block
+\startpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 2 \g
+ROLLBACK \bind \g
+\endpipeline
+-- Multiple explicit transactions
+\startpipeline
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+ROLLBACK \bind \g
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+COMMIT \bind \g
+\endpipeline
+-- COPY FROM STDIN
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+-- COPY FROM STDIN with \flushrequest + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\flushrequest
+\getresults
+ ?column?
+----------
+ val1
+(1 row)
+
+message type 0x5a arrived from server while idle
+\endpipeline
+-- COPY FROM STDIN with \syncpipeline + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\syncpipeline
+\getresults
+ ?column?
+----------
+ val1
+(1 row)
+
+\endpipeline
+-- COPY TO STDOUT
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+1 \N
+2 test2
+3 test3
+4 test4
+-- COPY TO STDOUT with \flushrequest + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\flushrequest
+\getresults
+ ?column?
+----------
+ val1
+(1 row)
+
+1 \N
+2 test2
+3 test3
+4 test4
+\endpipeline
+-- COPY TO STDOUT with \syncpipeline + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\syncpipeline
+\getresults
+ ?column?
+----------
+ val1
+(1 row)
+
+1 \N
+2 test2
+3 test3
+4 test4
+\endpipeline
+-- Use \parse and \bind_named
+\startpipeline
+SELECT $1 \parse ''
+SELECT $1, $2 \parse ''
+SELECT $2 \parse pipeline_1
+\bind_named '' 1 2 \g
+\bind_named pipeline_1 2 \g
+\endpipeline
+ERROR: could not determine data type of parameter $1
+-- \getresults displays all results preceding a \flushrequest
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+\flushrequest
+\getresults
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+\endpipeline
+-- \getresults displays all results preceding a sync
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+\syncpipeline
+\getresults
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+\endpipeline
+-- \getresults immediately returns if there's no result to fetch
+\startpipeline
+\getresults
+No pending results to get
+SELECT $1 \bind 2 \g
+\getresults
+No pending results to get
+\flushrequest
+\endpipeline
+ ?column?
+----------
+ 2
+(1 row)
+
+\getresults
+No pending results to get
+-- \getresults only fetch results preceding a flushrequest
+\startpipeline
+SELECT $1 \bind 2 \g
+\flushrequest
+SELECT $1 \bind 2 \g
+\getresults
+ ?column?
+----------
+ 2
+(1 row)
+
+\endpipeline
+ ?column?
+----------
+ 2
+(1 row)
+
+-- \getresults only fetch results preceding a sync message
+\startpipeline
+SELECT $1 \bind 2 \g
+\syncpipeline
+SELECT $1 \bind 2 \g
+\getresults
+ ?column?
+----------
+ 2
+(1 row)
+
+\endpipeline
+ ?column?
+----------
+ 2
+(1 row)
+
+-- use pipeline with chunked results with both \getresults and \endpipeline
+\startpipeline
+\set FETCH_COUNT 10
+SELECT $1 \bind 2 \g
+\flushrequest
+\getresults
+ ?column?
+----------
+ 2
+(1 row)
+
+SELECT $1 \bind 2 \g
+\endpipeline
+ ?column?
+----------
+ 2
+(1 row)
+
+\unset FETCH_COUNT
+-- \getresults with specific number of requested results
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+SELECT $1 \bind 3 \g
+\syncpipeline
+\getresults 1
+ ?column?
+----------
+ 1
+(1 row)
+
+SELECT $1 \bind 4 \g
+\getresults 3
+ ?column?
+----------
+ 2
+(1 row)
+
+ ?column?
+----------
+ 3
+(1 row)
+
+\endpipeline
+ ?column?
+----------
+ 4
+(1 row)
+
+-- \syncpipeline count as one command to fetch for \getresults
+\startpipeline
+\syncpipeline
+\syncpipeline
+SELECT $1 \bind 1 \g
+\flushrequest
+\getresults 2
+\getresults 1
+ ?column?
+----------
+ 1
+(1 row)
+
+\endpipeline
+-- \getresults 0 should get all results
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+SELECT $1 \bind 3 \g
+\syncpipeline
+\getresults 0
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+ ?column?
+----------
+ 3
+(1 row)
+
+\endpipeline
+-- pipelining errors
+-- endpipeline outside of pipeline should fail
+\endpipeline
+cannot send pipeline when not in pipeline mode
+-- Query using simple protocol should not be sent and should leave the pipeline usable
+\startpipeline
+SELECT 1;
+PQsendQuery not allowed in pipeline mode
+SELECT $1 \bind 'val1' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+-- After an aborted pipeline, commands after a sync should be displayed
+\startpipeline
+SELECT $1 \bind \g
+\syncpipeline
+SELECT $1 \bind 1 \g
+\endpipeline
+ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
+ ?column?
+----------
+ 1
+(1 row)
+
+-- Incorrect number of parameters, the pipeline will be aborted and following queries won't be executed
+\startpipeline
+SELECT \bind 'val1' \g
+SELECT $1 \bind 'val1' \g
+\endpipeline
+ERROR: bind message supplies 1 parameters, but prepared statement "" requires 0
+-- An explicit transaction with an error needs to be rollbacked after the pipeline
+\startpipeline
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+ROLLBACK \bind \g
+\endpipeline
+ERROR: duplicate key value violates unique constraint "psql_pipeline_pkey"
+DETAIL: Key (a)=(1) already exists.
+ROLLBACK;
+-- \watch sends a simple query which won't be allowed within a pipeline
+\startpipeline
+SELECT \bind \g
+\watch 1
+PQsendQuery not allowed in pipeline mode
+
+\endpipeline
+--
+(1 row)
+
+-- \gdesc should fail as synchronous commands are not allowed in pipeline, pipeline should still be usable
+\startpipeline
+SELECT $1 \bind 1 \gdesc
+synchronous command execution functions are not allowed in pipeline mode
+SELECT $1 \bind 1 \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+-- \gset is not allowed, pipeline should still be usable
+\startpipeline
+SELECT $1 as i, $2 as j \parse ''
+SELECT $1 as k, $2 as l \parse 'second'
+\bind_named '' 1 2 \gset
+\gset not allowed in pipeline mode
+\bind_named second 1 2 \gset pref02_ \echo :pref02_i :pref02_j
+\gset not allowed in pipeline mode
+\bind_named '' 1 2 \g
+\endpipeline
+ i | j
+---+---
+ 1 | 2
+(1 row)
+
+-- \gx is not allowed, pipeline should still be usable
+\startpipeline
+SELECT $1 \bind 1 \gx
+\gx not allowed in pipeline mode
+\reset
+SELECT $1 \bind 1 \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+-- \gx warning should be emitted in an aborted pipeline
+\startpipeline
+SELECT $1 \bind \g
+\flushrequest
+\getresults
+ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
+SELECT $1 \bind 1 \gx
+\gx not allowed in pipeline mode
+\endpipeline
+-- \gexec is not allowed, pipeline should still be usable
+\startpipeline
+SELECT 'INSERT INTO psql_pipeline(a) SELECT generate_series(1, 10)' \parse 'insert_stmt'
+\bind_named insert_stmt \gexec
+\gexec not allowed in pipeline mode
+\bind_named insert_stmt \g
+SELECT COUNT(*) FROM psql_pipeline \bind \g
+\endpipeline
+ ?column?
+------------------------------------------------------------
+ INSERT INTO psql_pipeline(a) SELECT generate_series(1, 10)
+(1 row)
+
+ count
+-------
+ 4
+(1 row)
+
+-- After an error, pipeline is aborted and requires a sync to be reusable
+\startpipeline
+SELECT $1 \bind \g
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+\flushrequest
+\getresults
+ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
+-- Pipeline is aborted
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+-- Sync allows pipeline to recover
+\syncpipeline
+\getresults
+Pipeline aborted, command didn't run
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+\flushrequest
+\getresults
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 1
+(1 row)
+
+\endpipeline
+-- In an aborted pipeline, \getresults 1 aborted commands one at a time
+\startpipeline
+SELECT $1 \bind \g
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\syncpipeline
+\getresults 1
+ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
+\getresults 1
+Pipeline aborted, command didn't run
+\getresults 1
+Pipeline aborted, command didn't run
+\getresults 1
+Pipeline aborted, command didn't run
+\getresults 1
+\endpipeline
+-- Test chunked results with an aborted pipeline
+\startpipeline
+\set FETCH_COUNT 10
+SELECT $1 \bind \g
+\flushrequest
+\getresults
+ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
+SELECT $1 \bind \g
+\endpipeline
+fetching results in chunked mode failed
+Pipeline aborted, command didn't run
+\unset FETCH_COUNT
+-- \getresults returns an error when an incorrect number is provided
+\startpipeline
+\getresults -1
+\getresults: invalid number of requested results
+\endpipeline
+-- \getresults when there's no result shouldn't impact the following query
+\getresults 1
+No pending results to get
+select 1;
+ ?column?
+----------
+ 1
+(1 row)
+
+-- pipelining and transaction block behaviour
+-- set local will issue a warning when modifying a GUC outside of a transaction block
+-- The change will still be valid as a pipeline runs within an implicit transaction block
+-- Sending a sync will commit the implicit transaction block. The first command after a sync
+-- won't be seen as belonging to a pipeline.
+\startpipeline
+SET LOCAL statement_timeout='1h' \bind \g
+SHOW statement_timeout \bind \g
+\syncpipeline
+SHOW statement_timeout \bind \g
+SET LOCAL statement_timeout='2h' \bind \g
+SHOW statement_timeout \bind \g
+\endpipeline
+WARNING: SET LOCAL can only be used in transaction blocks
+ statement_timeout
+-------------------
+ 1h
+(1 row)
+
+ statement_timeout
+-------------------
+ 0
+(1 row)
+
+ statement_timeout
+-------------------
+ 2h
+(1 row)
+
+-- Reindex concurrently is forbidden in the middle of a pipeline
+\startpipeline
+SELECT $1 \bind 1 \g
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ERROR: REINDEX CONCURRENTLY cannot run inside a transaction block
+-- Reindex concurrently will work if it's the first command of a pipeline
+\startpipeline
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ ?column?
+----------
+ 2
+(1 row)
+
+-- subtransactions are not allowed in pipeline mode
+\startpipeline
+SAVEPOINT a \bind \g
+SELECT $1 \bind 1 \g
+ROLLBACK TO SAVEPOINT a \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ERROR: SAVEPOINT can only be used in transaction blocks
+-- Lock command will fail as first pipeline command is not seen as a transaction block
+\startpipeline
+LOCK psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ERROR: LOCK TABLE can only be used in transaction blocks
+-- Lock command will succeed after the first command as pipeline will be seen as an implicit transaction block
+\startpipeline
+SELECT $1 \bind 1 \g
+LOCK psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+-- Vacuum command will work as the first command
+\startpipeline
+VACUUM psql_pipeline \bind \g
+\endpipeline
+-- Vacuum command will fail within pipeline implicit transaction
+\startpipeline
+SELECT 1 \bind \g
+VACUUM psql_pipeline \bind \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ERROR: VACUUM cannot run inside a transaction block
+-- Vacuum command will work after a sync
+\startpipeline
+SELECT 1 \bind \g
+\syncpipeline
+VACUUM psql_pipeline \bind \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
diff --git a/src/test/regress/sql/psql.sql b/src/test/regress/sql/psql.sql
index c5021fc0b13..33b204edd7f 100644
--- a/src/test/regress/sql/psql.sql
+++ b/src/test/regress/sql/psql.sql
@@ -1929,3 +1929,402 @@ CREATE TABLE defprivs (a int);
\z defprivs
\pset null ''
DROP TABLE defprivs;
+
+-- pipelining
+CREATE TABLE psql_pipeline(a INTEGER PRIMARY KEY, s TEXT);
+
+-- single query
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\endpipeline
+
+-- multiple queries
+\startpipeline
+SELECT $1 \bind 'val1' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\endpipeline
+
+-- Test \flush
+\startpipeline
+\flush
+SELECT $1 \bind 'val1' \g
+\flush
+SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\endpipeline
+
+-- send multiple syncs
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\syncpipeline
+\syncpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\syncpipeline
+SELECT $1, $2 \bind 'val4' 'val5' \g
+\endpipeline
+
+-- startpipeline shouldn't have any effect if already in a pipeline
+\startpipeline
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\endpipeline
+
+-- Convert an implicit tx block to an explicit tx block
+\startpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 2 \g
+ROLLBACK \bind \g
+\endpipeline
+
+-- Multiple explicit transactions
+\startpipeline
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+ROLLBACK \bind \g
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+COMMIT \bind \g
+\endpipeline
+
+-- COPY FROM STDIN
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\endpipeline
+2 test2
+\.
+
+-- COPY FROM STDIN with \flushrequest + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\flushrequest
+\getresults
+3 test3
+\.
+\endpipeline
+
+-- COPY FROM STDIN with \syncpipeline + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\syncpipeline
+\getresults
+4 test4
+\.
+\endpipeline
+
+-- COPY TO STDOUT
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\endpipeline
+
+-- COPY TO STDOUT with \flushrequest + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\flushrequest
+\getresults
+\endpipeline
+
+-- COPY TO STDOUT with \syncpipeline + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\syncpipeline
+\getresults
+\endpipeline
+
+-- Use \parse and \bind_named
+\startpipeline
+SELECT $1 \parse ''
+SELECT $1, $2 \parse ''
+SELECT $2 \parse pipeline_1
+\bind_named '' 1 2 \g
+\bind_named pipeline_1 2 \g
+\endpipeline
+
+-- \getresults displays all results preceding a \flushrequest
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+\flushrequest
+\getresults
+\endpipeline
+
+-- \getresults displays all results preceding a sync
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+\syncpipeline
+\getresults
+\endpipeline
+
+-- \getresults immediately returns if there's no result to fetch
+\startpipeline
+\getresults
+SELECT $1 \bind 2 \g
+\getresults
+\flushrequest
+\endpipeline
+\getresults
+
+-- \getresults only fetch results preceding a flushrequest
+\startpipeline
+SELECT $1 \bind 2 \g
+\flushrequest
+SELECT $1 \bind 2 \g
+\getresults
+\endpipeline
+
+-- \getresults only fetch results preceding a sync message
+\startpipeline
+SELECT $1 \bind 2 \g
+\syncpipeline
+SELECT $1 \bind 2 \g
+\getresults
+\endpipeline
+
+-- use pipeline with chunked results with both \getresults and \endpipeline
+\startpipeline
+\set FETCH_COUNT 10
+SELECT $1 \bind 2 \g
+\flushrequest
+\getresults
+SELECT $1 \bind 2 \g
+\endpipeline
+\unset FETCH_COUNT
+
+-- \getresults with specific number of requested results
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+SELECT $1 \bind 3 \g
+\syncpipeline
+\getresults 1
+SELECT $1 \bind 4 \g
+\getresults 3
+\endpipeline
+
+-- \syncpipeline count as one command to fetch for \getresults
+\startpipeline
+\syncpipeline
+\syncpipeline
+SELECT $1 \bind 1 \g
+\flushrequest
+\getresults 2
+\getresults 1
+\endpipeline
+
+-- \getresults 0 should get all results
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+SELECT $1 \bind 3 \g
+\syncpipeline
+\getresults 0
+\endpipeline
+
+-- pipelining errors
+
+-- endpipeline outside of pipeline should fail
+\endpipeline
+
+-- Query using simple protocol should not be sent and should leave the pipeline usable
+\startpipeline
+SELECT 1;
+SELECT $1 \bind 'val1' \g
+\endpipeline
+
+-- After an aborted pipeline, commands after a sync should be displayed
+\startpipeline
+SELECT $1 \bind \g
+\syncpipeline
+SELECT $1 \bind 1 \g
+\endpipeline
+
+-- Incorrect number of parameters, the pipeline will be aborted and following queries won't be executed
+\startpipeline
+SELECT \bind 'val1' \g
+SELECT $1 \bind 'val1' \g
+\endpipeline
+
+-- An explicit transaction with an error needs to be rollbacked after the pipeline
+\startpipeline
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+ROLLBACK \bind \g
+\endpipeline
+ROLLBACK;
+
+-- \watch sends a simple query which won't be allowed within a pipeline
+\startpipeline
+SELECT \bind \g
+\watch 1
+\endpipeline
+
+-- \gdesc should fail as synchronous commands are not allowed in pipeline, pipeline should still be usable
+\startpipeline
+SELECT $1 \bind 1 \gdesc
+SELECT $1 \bind 1 \g
+\endpipeline
+
+-- \gset is not allowed, pipeline should still be usable
+\startpipeline
+SELECT $1 as i, $2 as j \parse ''
+SELECT $1 as k, $2 as l \parse 'second'
+\bind_named '' 1 2 \gset
+\bind_named second 1 2 \gset pref02_ \echo :pref02_i :pref02_j
+\bind_named '' 1 2 \g
+\endpipeline
+
+-- \gx is not allowed, pipeline should still be usable
+\startpipeline
+SELECT $1 \bind 1 \gx
+\reset
+SELECT $1 \bind 1 \g
+\endpipeline
+
+-- \gx warning should be emitted in an aborted pipeline
+\startpipeline
+SELECT $1 \bind \g
+\flushrequest
+\getresults
+SELECT $1 \bind 1 \gx
+\endpipeline
+
+-- \gexec is not allowed, pipeline should still be usable
+\startpipeline
+SELECT 'INSERT INTO psql_pipeline(a) SELECT generate_series(1, 10)' \parse 'insert_stmt'
+\bind_named insert_stmt \gexec
+\bind_named insert_stmt \g
+SELECT COUNT(*) FROM psql_pipeline \bind \g
+\endpipeline
+
+-- After an error, pipeline is aborted and requires a sync to be reusable
+\startpipeline
+SELECT $1 \bind \g
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+\flushrequest
+\getresults
+-- Pipeline is aborted
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+-- Sync allows pipeline to recover
+\syncpipeline
+\getresults
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+\flushrequest
+\getresults
+\endpipeline
+
+-- In an aborted pipeline, \getresults 1 aborted commands one at a time
+\startpipeline
+SELECT $1 \bind \g
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\syncpipeline
+\getresults 1
+\getresults 1
+\getresults 1
+\getresults 1
+\getresults 1
+\endpipeline
+
+-- Test chunked results with an aborted pipeline
+\startpipeline
+\set FETCH_COUNT 10
+SELECT $1 \bind \g
+\flushrequest
+\getresults
+SELECT $1 \bind \g
+\endpipeline
+\unset FETCH_COUNT
+
+-- \getresults returns an error when an incorrect number is provided
+\startpipeline
+\getresults -1
+\endpipeline
+
+-- \getresults when there's no result shouldn't impact the following query
+\getresults 1
+select 1;
+
+-- pipelining and transaction block behaviour
+
+-- set local will issue a warning when modifying a GUC outside of a transaction block
+-- The change will still be valid as a pipeline runs within an implicit transaction block
+-- Sending a sync will commit the implicit transaction block. The first command after a sync
+-- won't be seen as belonging to a pipeline.
+\startpipeline
+SET LOCAL statement_timeout='1h' \bind \g
+SHOW statement_timeout \bind \g
+\syncpipeline
+SHOW statement_timeout \bind \g
+SET LOCAL statement_timeout='2h' \bind \g
+SHOW statement_timeout \bind \g
+\endpipeline
+
+-- Reindex concurrently is forbidden in the middle of a pipeline
+\startpipeline
+SELECT $1 \bind 1 \g
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- Reindex concurrently will work if it's the first command of a pipeline
+\startpipeline
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- subtransactions are not allowed in pipeline mode
+\startpipeline
+SAVEPOINT a \bind \g
+SELECT $1 \bind 1 \g
+ROLLBACK TO SAVEPOINT a \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- Lock command will fail as first pipeline command is not seen as a transaction block
+\startpipeline
+LOCK psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- Lock command will succeed after the first command as pipeline will be seen as an implicit transaction block
+\startpipeline
+SELECT $1 \bind 1 \g
+LOCK psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- Vacuum command will work as the first command
+\startpipeline
+VACUUM psql_pipeline \bind \g
+\endpipeline
+
+-- Vacuum command will fail within pipeline implicit transaction
+\startpipeline
+SELECT 1 \bind \g
+VACUUM psql_pipeline \bind \g
+\endpipeline
+
+-- Vacuum command will work after a sync
+\startpipeline
+SELECT 1 \bind \g
+\syncpipeline
+VACUUM psql_pipeline \bind \g
+\endpipeline
--
2.39.5 (Apple Git-154)
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-01-14 08:49 Anthonin Bonnefoy <[email protected]>
parent: Anthonin Bonnefoy <[email protected]>
0 siblings, 1 reply; 41+ messages in thread
From: Anthonin Bonnefoy @ 2025-01-14 08:49 UTC (permalink / raw)
To: Jelte Fennema-Nio <[email protected]>; +Cc: Michael Paquier <[email protected]>; pgsql-hackers
During my tests, I've noticed I didn't handle query cancellation, this
is now fixed. I've also added additional comments related to
available_results to make it clearer that it depends on what the
server has flushed to the client.
Attachments:
[application/octet-stream] v07-0001-Add-pipelining-support-in-psql.patch (55.9K, ../../CAO6_XqppNRbiRDF608ZogmLLtwdA8ENak0QAy40fDM5U_zqrHA@mail.gmail.com/2-v07-0001-Add-pipelining-support-in-psql.patch)
download | inline diff:
From ec2dfa63e3be485d6700c56ff4fe0a64d47181ea Mon Sep 17 00:00:00 2001
From: Anthonin Bonnefoy <[email protected]>
Date: Tue, 5 Nov 2024 10:26:54 +0100
Subject: Add pipelining support in psql
With \bind, \parse, \bind_named and \close, it is possible to issue
queries from psql using the extended protocol. However, it wasn't
possible to send those queries using pipelining and the only way to test
pipelined queries was through pgbench's tap tests.
This patch adds additional psql meta-commands to support pipelining:
\startpipeline, \endpipeline and \syncpipeline, mirroring the existing
meta-commands in pgbench. Additional meta-commands allow to flush and
read results of an ongoing pipeline: \flushrequest, \flush and \getresults
\startpipeline starts a new pipeline. All extended queries will be
queued until the end of the pipeline is reached.
\endpipeline ends an ongoing pipeline. All queued commands will be sent
to the server and all responses will be processed by the psql.
\syncpipeline queues a synchronisation point without flushing the
commands to the server
\flush Call PQflush on psql's connection
\flushrequest queues a flushrequest
\getresults reads server's results. Unsent data are automatically pushed
when \getresults is called
Those meta-commands will allow to test pipeline behaviour using
psql regression tests.
---
doc/src/sgml/ref/psql-ref.sgml | 96 +++++
src/bin/psql/command.c | 151 +++++++
src/bin/psql/common.c | 274 +++++++++++-
src/bin/psql/help.c | 7 +
src/bin/psql/prompt.c | 12 +
src/bin/psql/settings.h | 16 +-
src/bin/psql/tab-complete.in.c | 8 +-
src/test/regress/expected/psql.out | 668 +++++++++++++++++++++++++++++
src/test/regress/sql/psql.sql | 399 +++++++++++++++++
9 files changed, 1619 insertions(+), 12 deletions(-)
diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml
index 72f3347e53d..3bc77d2bb0e 100644
--- a/doc/src/sgml/ref/psql-ref.sgml
+++ b/doc/src/sgml/ref/psql-ref.sgml
@@ -3562,6 +3562,82 @@ testdb=> <userinput>\setenv LESS -imx4F</userinput>
</listitem>
</varlistentry>
+ <varlistentry id="app-psql-meta-command-pipeline">
+ <term><literal>\startpipeline</literal></term>
+ <term><literal>\syncpipeline</literal></term>
+ <term><literal>\endpipeline</literal></term>
+ <term><literal>\flushrequest</literal></term>
+ <term><literal>\flush</literal></term>
+ <term><literal>\getresults [ <replaceable class="parameter">number_results</replaceable> ]</literal></term>
+
+ <listitem>
+ <para>
+ This group of commands implements pipelining of SQL statements.
+ A pipeline must begin with a <command>\startpipeline</command>
+ and end with an <command>\endpipeline</command>. In between there
+ may be any number of <command>\syncpipeline</command> commands,
+ which sends a <link linkend="protocol-flow-ext-query">sync message</link>
+ without ending the ongoing pipeline and flushing the send buffer.
+ In pipeline mode, statements are sent to the server without waiting
+ for the results of previous statements. See
+ <xref linkend="libpq-pipeline-mode"/> for more details.
+ </para>
+
+ <para>
+ Pipeline mode requires the use of extended query protocol. All queries need
+ to be sent using the meta-commands <literal>\bind</literal>,
+ <literal>\bind_named</literal>, <literal>\close</literal> or
+ <literal>\parse</literal>. While a pipeline is ongoing,
+ <literal>\g</literal> will append the current query buffer to the pipeline and
+ other meta-commands like <literal>\gx</literal> or <literal>\gdesc</literal>
+ are not allowed in pipeline mode.
+ </para>
+
+ <para>
+ <command>\flushrequest</command> appends a flush command to the pipeline,
+ allowing to read results with <command>\getresults</command> without issuing
+ a sync or ending the pipeline. <command>\getresults</command> will automatically
+ push unsent data to the server. <command>\flush</command> can be used to manually
+ push unsent data.
+ </para>
+
+ <para>
+ <command>\getresults</command> accepts an optional
+ <replaceable class="parameter">number_results</replaceable> parameter. If provided,
+ only the first <replaceable class="parameter">number_results</replaceable> pending
+ results will be read. If not provided or 0, all pending results are read. The
+ commands <literal>\bind</literal>, <literal>\bind_named</literal>,
+ <literal>\close</literal>, <literal>\parse</literal> and <literal>\syncpipeline</literal>
+ generate one result to get.
+ </para>
+
+ <para>
+ When pipeline mode is active, a dedicated prompt variable is
+ available to report the pipeline status. See
+ <xref linkend="app-psql-prompting-p-uc"/> for more details
+ </para>
+
+ <para>
+ Example:
+<programlisting>
+\startpipeline
+SELECT 1 \bind \g
+SELECT $1 \parse stmt1
+\bind_named stmt1 1 \g
+SELECT pg_current_xact_id() \bind \g
+\flushrequest
+\getresults
+\syncpipeline
+SELECT pg_current_xact_id() \bind \g
+\flushrequest
+\getresults 2
+\close stmt1
+\endpipeline
+</programlisting></para>
+
+ </listitem>
+ </varlistentry>
+
<varlistentry id="app-psql-meta-command-t-lc">
<term><literal>\t</literal></term>
@@ -4719,6 +4795,26 @@ testdb=> <userinput>INSERT INTO my_table VALUES (:'content');</userinput>
</listitem>
</varlistentry>
+ <varlistentry id="app-psql-prompting-p-uc">
+ <term><literal>%P</literal></term>
+ <listitem>
+ <para>
+ Pipeline status: an empty string when not in a pipeline,
+ or <literal>|piped_syncs,piped_commands,pending_results|</literal>
+ when in an ongoing pipeline, or <literal>+piped_syncs,piped_commands,pending_results+</literal>
+ when in an aborted pipeline. <literal>piped_syncs</literal> is the
+ number of syncs queued in the pipeline. <literal>piped_commands</literal>
+ is the number of commands generated by <literal>\bind</literal>,
+ <literal>\bind_named</literal>, <literal>\close</literal> or
+ <literal>\parse</literal> queued in the pipeline. <literal>pending_results</literal>
+ is the number of commands that were followed by either a
+ <command>\flushrequest</command> or a <command>\syncpipeline</command>,
+ forcing the server to send the results that can be retrieved with
+ <command>\getresults</command>.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry id="app-psql-prompting-r">
<term><literal>%R</literal></term>
<listitem>
diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c
index 5dd4c2d2687..47682353fc6 100644
--- a/src/bin/psql/command.c
+++ b/src/bin/psql/command.c
@@ -90,9 +90,12 @@ static backslashResult exec_command_else(PsqlScanState scan_state, ConditionalSt
PQExpBuffer query_buf);
static backslashResult exec_command_endif(PsqlScanState scan_state, ConditionalStack cstack,
PQExpBuffer query_buf);
+static backslashResult exec_command_endpipeline(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_encoding(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_errverbose(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_f(PsqlScanState scan_state, bool active_branch);
+static backslashResult exec_command_flush(PsqlScanState scan_state, bool active_branch);
+static backslashResult exec_command_flushrequest(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_g(PsqlScanState scan_state, bool active_branch,
const char *cmd);
static backslashResult process_command_g_options(char *first_option,
@@ -103,6 +106,7 @@ static backslashResult exec_command_gdesc(PsqlScanState scan_state, bool active_
static backslashResult exec_command_getenv(PsqlScanState scan_state, bool active_branch,
const char *cmd);
static backslashResult exec_command_gexec(PsqlScanState scan_state, bool active_branch);
+static backslashResult exec_command_getresults(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_gset(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_help(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_html(PsqlScanState scan_state, bool active_branch);
@@ -132,6 +136,8 @@ static backslashResult exec_command_setenv(PsqlScanState scan_state, bool active
const char *cmd);
static backslashResult exec_command_sf_sv(PsqlScanState scan_state, bool active_branch,
const char *cmd, bool is_func);
+static backslashResult exec_command_startpipeline(PsqlScanState scan_state, bool active_branch);
+static backslashResult exec_command_syncpipeline(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_t(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_T(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_timing(PsqlScanState scan_state, bool active_branch);
@@ -351,18 +357,26 @@ exec_command(const char *cmd,
status = exec_command_else(scan_state, cstack, query_buf);
else if (strcmp(cmd, "endif") == 0)
status = exec_command_endif(scan_state, cstack, query_buf);
+ else if (strcmp(cmd, "endpipeline") == 0)
+ status = exec_command_endpipeline(scan_state, active_branch);
else if (strcmp(cmd, "encoding") == 0)
status = exec_command_encoding(scan_state, active_branch);
else if (strcmp(cmd, "errverbose") == 0)
status = exec_command_errverbose(scan_state, active_branch);
else if (strcmp(cmd, "f") == 0)
status = exec_command_f(scan_state, active_branch);
+ else if (strcmp(cmd, "flush") == 0)
+ status = exec_command_flush(scan_state, active_branch);
+ else if (strcmp(cmd, "flushrequest") == 0)
+ status = exec_command_flushrequest(scan_state, active_branch);
else if (strcmp(cmd, "g") == 0 || strcmp(cmd, "gx") == 0)
status = exec_command_g(scan_state, active_branch, cmd);
else if (strcmp(cmd, "gdesc") == 0)
status = exec_command_gdesc(scan_state, active_branch);
else if (strcmp(cmd, "getenv") == 0)
status = exec_command_getenv(scan_state, active_branch, cmd);
+ else if (strcmp(cmd, "getresults") == 0)
+ status = exec_command_getresults(scan_state, active_branch);
else if (strcmp(cmd, "gexec") == 0)
status = exec_command_gexec(scan_state, active_branch);
else if (strcmp(cmd, "gset") == 0)
@@ -408,6 +422,10 @@ exec_command(const char *cmd,
status = exec_command_sf_sv(scan_state, active_branch, cmd, true);
else if (strcmp(cmd, "sv") == 0 || strcmp(cmd, "sv+") == 0)
status = exec_command_sf_sv(scan_state, active_branch, cmd, false);
+ else if (strcmp(cmd, "startpipeline") == 0)
+ status = exec_command_startpipeline(scan_state, active_branch);
+ else if (strcmp(cmd, "syncpipeline") == 0)
+ status = exec_command_syncpipeline(scan_state, active_branch);
else if (strcmp(cmd, "t") == 0)
status = exec_command_t(scan_state, active_branch);
else if (strcmp(cmd, "T") == 0)
@@ -1491,6 +1509,38 @@ exec_command_f(PsqlScanState scan_state, bool active_branch)
return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR;
}
+/*
+ * \flush -- call PQflush on the connection
+ */
+static backslashResult
+exec_command_flush(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_FLUSH;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
+/*
+ * \flushrequest -- send a flush request to the server
+ */
+static backslashResult
+exec_command_flushrequest(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_SEND_FLUSH_REQUEST;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
/*
* \g [(pset-option[=pset-value] ...)] [filename/shell-command]
* \gx [(pset-option[=pset-value] ...)] [filename/shell-command]
@@ -1526,6 +1576,13 @@ exec_command_g(PsqlScanState scan_state, bool active_branch, const char *cmd)
if (status == PSQL_CMD_SKIP_LINE && active_branch)
{
+ if (strcmp(cmd, "gx") == 0 && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pg_log_error("\\gx not allowed in pipeline mode");
+ clean_extended_state();
+ return PSQL_CMD_ERROR;
+ }
+
if (!fname)
pset.gfname = NULL;
else
@@ -1679,6 +1736,39 @@ exec_command_getenv(PsqlScanState scan_state, bool active_branch,
return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR;
}
+/*
+ * \getresults -- read results
+ */
+static backslashResult
+exec_command_getresults(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ char *opt;
+ int num_results;
+
+ pset.send_mode = PSQL_GET_RESULTS;
+ opt = psql_scan_slash_option(scan_state, OT_NORMAL, NULL, false);
+
+ pset.requested_results = 0;
+ if (opt != NULL)
+ {
+ num_results = atoi(opt);
+ if (num_results < 0)
+ {
+ pg_log_error("\\getresults: invalid number of requested results");
+ return PSQL_CMD_SKIP_LINE;
+ }
+ pset.requested_results = num_results;
+ }
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
+
/*
* \gexec -- send query and execute each field of result
*/
@@ -1689,6 +1779,12 @@ exec_command_gexec(PsqlScanState scan_state, bool active_branch)
if (active_branch)
{
+ if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pg_log_error("\\gexec not allowed in pipeline mode");
+ clean_extended_state();
+ return PSQL_CMD_ERROR;
+ }
pset.gexec_flag = true;
status = PSQL_CMD_SEND;
}
@@ -1709,6 +1805,13 @@ exec_command_gset(PsqlScanState scan_state, bool active_branch)
char *prefix = psql_scan_slash_option(scan_state,
OT_NORMAL, NULL, false);
+ if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pg_log_error("\\gset not allowed in pipeline mode");
+ clean_extended_state();
+ return PSQL_CMD_ERROR;
+ }
+
if (prefix)
pset.gset_prefix = prefix;
else
@@ -2672,6 +2775,54 @@ exec_command_sf_sv(PsqlScanState scan_state, bool active_branch,
return status;
}
+/*
+ * \startpipeline -- enter pipeline mode
+ */
+static backslashResult
+exec_command_startpipeline(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_START_PIPELINE_MODE;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
+/*
+ * \syncpipeline -- send a sync message to an active pipeline
+ */
+static backslashResult
+exec_command_syncpipeline(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_SEND_PIPELINE_SYNC;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
+/*
+ * \endpipeline -- end pipeline mode
+ */
+static backslashResult
+exec_command_endpipeline(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_END_PIPELINE_MODE;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
/*
* \t -- turn off table headers and row count
*/
diff --git a/src/bin/psql/common.c b/src/bin/psql/common.c
index f1a5291c13b..e1b58f08c59 100644
--- a/src/bin/psql/common.c
+++ b/src/bin/psql/common.c
@@ -121,6 +121,18 @@ CloseGOutput(FILE *gfile_fout, bool is_pipe)
}
}
+/*
+ * Reset pset pipeline state
+ */
+static void
+pipelineReset(void)
+{
+ pset.piped_syncs = 0;
+ pset.piped_commands = 0;
+ pset.available_results = 0;
+ pset.requested_results = 0;
+}
+
/*
* setQFout
* -- handler for -o command line option and \o command
@@ -354,6 +366,7 @@ CheckConnection(void)
fprintf(stderr, _("The connection to the server was lost. Attempting reset: "));
PQreset(pset.db);
+ pipelineReset();
OK = ConnectionUp();
if (!OK)
{
@@ -415,10 +428,12 @@ AcceptResult(const PGresult *result, bool show_error)
case PGRES_EMPTY_QUERY:
case PGRES_COPY_IN:
case PGRES_COPY_OUT:
+ case PGRES_PIPELINE_SYNC:
/* Fine, do nothing */
OK = true;
break;
+ case PGRES_PIPELINE_ABORTED:
case PGRES_BAD_RESPONSE:
case PGRES_NONFATAL_ERROR:
case PGRES_FATAL_ERROR:
@@ -1050,6 +1065,7 @@ PrintQueryResult(PGresult *result, bool last,
success = true;
break;
+ case PGRES_PIPELINE_ABORTED:
case PGRES_BAD_RESPONSE:
case PGRES_NONFATAL_ERROR:
case PGRES_FATAL_ERROR:
@@ -1418,6 +1434,60 @@ DescribeQuery(const char *query, double *elapsed_msec)
return OK;
}
+/*
+ * Read and discard all results in an aborted pipeline.
+ *
+ * If a synchronisation point is found, we can stop discarding results as
+ * pipeline will switch back to an OK state. If no synchronisation point
+ * is available, we need to stop when there's no more pending results,
+ * otherwise, calling PQgetResults will block.
+ */
+static PGresult *
+discardAbortedPipelineResults(void)
+{
+ for (;;)
+ {
+ PGresult *res = PQgetResult(pset.db);
+ ExecStatusType result_status = PQresultStatus(res);
+
+ if (result_status == PGRES_PIPELINE_SYNC)
+ {
+ /*
+ * Found a synchronisation point. Decrementing the sync counter
+ * will be done by the caller
+ */
+ return res;
+ }
+ else if (res == NULL)
+ {
+ /* A query was processed, decrement the counters */
+ Assert(pset.available_results > 0);
+ Assert(pset.requested_results > 0);
+ pset.available_results--;
+ pset.requested_results--;
+ }
+
+ if (pset.requested_results == 0)
+ /* We've read all requested results, exit */
+ return res;
+
+ if (pset.available_results == 0 && pset.piped_syncs == 0)
+
+ /*
+ * There's no more results to get and there's no synchronisation
+ * point to stop at. This will leave the pipeline in an aborted
+ * state.
+ */
+ return res;
+
+ /*
+ * An aborted pipeline will have either NULL results or results in an
+ * PGRES_PIPELINE_ABORTED status
+ */
+ Assert(res == NULL || result_status == PGRES_PIPELINE_ABORTED);
+ PQclear(res);
+ }
+}
/*
* ExecQueryAndProcessResults: utility function for use by SendQuery()
@@ -1451,6 +1521,7 @@ ExecQueryAndProcessResults(const char *query,
bool timing = pset.timing;
bool success = false;
bool return_early = false;
+ bool end_pipeline = false;
instr_time before,
after;
PGresult *result;
@@ -1466,9 +1537,13 @@ ExecQueryAndProcessResults(const char *query,
{
case PSQL_SEND_EXTENDED_CLOSE:
success = PQsendClosePrepared(pset.db, pset.stmtName);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ pset.piped_commands++;
break;
case PSQL_SEND_EXTENDED_PARSE:
success = PQsendPrepare(pset.db, pset.stmtName, query, 0, NULL);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ pset.piped_commands++;
break;
case PSQL_SEND_EXTENDED_QUERY_PARAMS:
Assert(pset.stmtName == NULL);
@@ -1476,6 +1551,8 @@ ExecQueryAndProcessResults(const char *query,
pset.bind_nparams, NULL,
(const char *const *) pset.bind_params,
NULL, NULL, 0);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ pset.piped_commands++;
break;
case PSQL_SEND_EXTENDED_QUERY_PREPARED:
Assert(pset.stmtName != NULL);
@@ -1483,6 +1560,83 @@ ExecQueryAndProcessResults(const char *query,
pset.bind_nparams,
(const char *const *) pset.bind_params,
NULL, NULL, 0);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ pset.piped_commands++;
+ break;
+ case PSQL_START_PIPELINE_MODE:
+ success = PQenterPipelineMode(pset.db);
+ break;
+ case PSQL_END_PIPELINE_MODE:
+ success = PQpipelineSync(pset.db);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ /*
+ * End of the pipeline, all queued commands need to be
+ * processed
+ */
+ end_pipeline = true;
+ pset.piped_syncs++;
+
+ /*
+ * The server will send a ReadyForQuery after a Sync is
+ * processed, flushing all results to the client
+ */
+ pset.available_results += pset.piped_commands;
+ pset.piped_commands = 0;
+ /* We want to read all results */
+ pset.requested_results = pset.available_results + pset.piped_syncs;
+ }
+ break;
+ case PSQL_SEND_PIPELINE_SYNC:
+ success = PQsendPipelineSync(pset.db);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pset.piped_syncs++;
+
+ /*
+ * The server will send a ReadyForQuery after a Sync is
+ * processed, flushing all results to the client
+ */
+ pset.available_results += pset.piped_commands;
+ pset.piped_commands = 0;
+ }
+ break;
+ case PSQL_FLUSH:
+ success = PQflush(pset.db);
+ break;
+ case PSQL_SEND_FLUSH_REQUEST:
+ success = PQsendFlushRequest(pset.db);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ /*
+ * With the flush request, all piped commands are pushed and
+ * the server will forcefully flush the results to the client,
+ * making them available
+ */
+ pset.available_results += pset.piped_commands;
+ pset.piped_commands = 0;
+ }
+ break;
+ case PSQL_GET_RESULTS:
+ if (pset.available_results == 0 && pset.piped_syncs == 0)
+ {
+ /*
+ * If no sync or flush request were sent, PQgetResult will
+ * block. Forbid the call to \getresults to avoid staying
+ * stuck
+ */
+ pg_log_info("No pending results to get");
+ success = false;
+ pset.requested_results = 0;
+ }
+ else
+ {
+ success = true;
+ /* Cap requested_results to the maximum known results */
+ if (pset.requested_results == 0 ||
+ pset.requested_results > (pset.available_results + pset.piped_syncs))
+ pset.requested_results = pset.available_results + pset.piped_syncs;
+ }
break;
case PSQL_SEND_QUERY:
success = PQsendQuery(pset.db, query);
@@ -1501,6 +1655,16 @@ ExecQueryAndProcessResults(const char *query,
return -1;
}
+ if (pset.requested_results == 0 && !end_pipeline &&
+ PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ /*
+ * We're in a pipeline and haven't reached the pipeline end or there
+ * was no request to read pipeline results, exit.
+ */
+ return 1;
+ }
+
/*
* Fetch the result in chunks if FETCH_COUNT is set, except when:
*
@@ -1548,7 +1712,7 @@ ExecQueryAndProcessResults(const char *query,
{
ExecStatusType result_status;
bool is_chunked_result = false;
- PGresult *next_result;
+ PGresult *next_result = NULL;
bool last;
if (!AcceptResult(result, false))
@@ -1571,6 +1735,9 @@ ExecQueryAndProcessResults(const char *query,
ClearOrSaveResult(result);
success = false;
+ if (result_status == PGRES_PIPELINE_ABORTED)
+ pg_log_info("Pipeline aborted, command didn't run");
+
/*
* switch to next result
*/
@@ -1585,6 +1752,20 @@ ExecQueryAndProcessResults(const char *query,
* ignore manually.
*/
result = NULL;
+ else if ((end_pipeline || pset.requested_results > 0)
+ && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+
+ /*
+ * We have an error within a pipeline. All commands are
+ * aborted until the next synchronisation point. We need to
+ * consume all results until this synchronisation point, or
+ * stop when there's no more result to discard
+ *
+ * Checking pipeline status is necessary in case the
+ * connection was reset. The new connection isn't in any kind
+ * of pipeline state and thus has no result to discard
+ */
+ result = discardAbortedPipelineResults();
else
result = PQgetResult(pset.db);
@@ -1771,12 +1952,66 @@ ExecQueryAndProcessResults(const char *query,
}
}
+ if (result_status == PGRES_PIPELINE_SYNC)
+ {
+ Assert(pset.piped_syncs > 0);
+
+ /*
+ * We have a sync response, decrease the sync and
+ * requested_results counters
+ */
+ pset.piped_syncs--;
+ pset.requested_results--;
+
+ /*
+ * After a synchronisation point, reset success state to print
+ * possible successful results that will be processed after this
+ */
+ success = true;
+
+ /*
+ * If all syncs were processed and pipeline end was requested,
+ * exit pipeline mode
+ */
+ if (end_pipeline && pset.piped_syncs == 0)
+ success &= PQexitPipelineMode(pset.db);
+ }
+ else if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF &&
+ result_status != PGRES_PIPELINE_SYNC)
+ {
+ /*
+ * We are in a pipeline and have a non sync response, decrease the
+ * results counters
+ */
+ pset.available_results--;
+ pset.requested_results--;
+ }
+
/*
* Check PQgetResult() again. In the typical case of a single-command
* string, it will return NULL. Otherwise, we'll have other results
* to process. We need to do that to check whether this is the last.
*/
- next_result = PQgetResult(pset.db);
+ if (PQpipelineStatus(pset.db) == PQ_PIPELINE_OFF)
+ next_result = PQgetResult(pset.db);
+ else
+ {
+ /*
+ * In pipeline mode, a NULL result indicates the end of the
+ * current query being processed. Call PQgetResult to consume this
+ * NULL
+ */
+ if (result_status != PGRES_PIPELINE_SYNC)
+ {
+ next_result = PQgetResult(pset.db);
+ Assert(next_result == NULL);
+ }
+
+ /* We can now get the next result in the pipeline */
+ if (pset.requested_results > 0)
+ next_result = PQgetResult(pset.db);
+ }
+
last = (next_result == NULL);
/*
@@ -1798,8 +2033,12 @@ ExecQueryAndProcessResults(const char *query,
*elapsed_msec = INSTR_TIME_GET_MILLISEC(after);
}
- /* this may or may not print something depending on settings */
- if (result != NULL)
+ /*
+ * This may or may not print something depending on settings. A
+ * pipeline sync will have a non null result but doesn't have anything
+ * to print, thus ignore them
+ */
+ if (result != NULL && result_status != PGRES_PIPELINE_SYNC)
{
/*
* If results need to be printed into the file specified by \g,
@@ -1825,9 +2064,15 @@ ExecQueryAndProcessResults(const char *query,
ClearOrSaveResult(result);
result = next_result;
- if (cancel_pressed)
+ if (cancel_pressed && PQpipelineStatus(pset.db) == PQ_PIPELINE_OFF)
{
- /* drop this next result, as well as any others not yet read */
+ /*
+ * Outside of a pipeline, drop this next result, as well as any
+ * others not yet read
+ *
+ * Within a pipeline, we can let the outer loop handle this as an
+ * aborted pipeline, which will discard all results
+ */
ClearOrSaveResult(result);
ClearOrSaveAllResults();
break;
@@ -1837,6 +2082,17 @@ ExecQueryAndProcessResults(const char *query,
/* close \g file if we opened it */
CloseGOutput(gfile_fout, gfile_is_pipe);
+ if (end_pipeline)
+ {
+ /* After a pipeline is processed, pipeline piped_syncs should be 0 */
+ Assert(pset.piped_syncs == 0);
+ /* all commands were processed */
+ Assert(pset.piped_commands == 0);
+ /* and all results were read */
+ Assert(pset.available_results == 0);
+ }
+ Assert(pset.requested_results == 0);
+
/* may need this to recover from conn loss during COPY */
if (!CheckConnection())
return -1;
@@ -2296,7 +2552,13 @@ clean_extended_state(void)
free(pset.stmtName);
pset.bind_params = NULL;
break;
+ case PSQL_START_PIPELINE_MODE: /* \startpipeline */
+ case PSQL_END_PIPELINE_MODE: /* \endpipeline */
+ case PSQL_SEND_PIPELINE_SYNC: /* \syncpipeline */
case PSQL_SEND_QUERY:
+ case PSQL_FLUSH: /* \flush */
+ case PSQL_GET_RESULTS: /* \getresults */
+ case PSQL_SEND_FLUSH_REQUEST: /* \flushrequest */
break;
}
diff --git a/src/bin/psql/help.c b/src/bin/psql/help.c
index fda83465efa..92df53ffd67 100644
--- a/src/bin/psql/help.c
+++ b/src/bin/psql/help.c
@@ -167,15 +167,22 @@ slashUsage(unsigned short int pager)
HELP0(" \\close STMT_NAME close an existing prepared statement\n");
HELP0(" \\copyright show PostgreSQL usage and distribution terms\n");
HELP0(" \\crosstabview [COLUMNS] execute query and display result in crosstab\n");
+ HELP0(" \\endpipeline exit pipeline mode\n");
HELP0(" \\errverbose show most recent error message at maximum verbosity\n");
+ HELP0(" \\flush push unsent data to the server\n");
+ HELP0(" \\flushrequest send a flushrequest command\n");
HELP0(" \\g [(OPTIONS)] [FILE] execute query (and send result to file or |pipe);\n"
" \\g with no arguments is equivalent to a semicolon\n");
HELP0(" \\gdesc describe result of query, without executing it\n");
+ HELP0(" \\getresults [NUM_RES] read NUM_RES pending results. All pending results are\n"
+ " read if no argument is provided\n");
HELP0(" \\gexec execute query, then execute each value in its result\n");
HELP0(" \\gset [PREFIX] execute query and store result in psql variables\n");
HELP0(" \\gx [(OPTIONS)] [FILE] as \\g, but forces expanded output mode\n");
HELP0(" \\parse STMT_NAME create a prepared statement\n");
HELP0(" \\q quit psql\n");
+ HELP0(" \\startpipeline enter pipeline mode\n");
+ HELP0(" \\syncpipeline add a synchronisation point to an ongoing pipeline\n");
HELP0(" \\watch [[i=]SEC] [c=N] [m=MIN]\n"
" execute query every SEC seconds, up to N times,\n"
" stop if less than MIN rows are returned\n");
diff --git a/src/bin/psql/prompt.c b/src/bin/psql/prompt.c
index 08a14feb3c3..d8e271b82a1 100644
--- a/src/bin/psql/prompt.c
+++ b/src/bin/psql/prompt.c
@@ -31,6 +31,9 @@
* sockets, "[local:/dir/name]" if not default
* %m - like %M, but hostname only (before first dot), or always "[local]"
* %p - backend pid
+ * %P - pipeline status: no pipeline: empty
+ * ongoing pipeline: |piped_syncs,piped_commands,pending_results|
+ * aborted pipeline: +piped_syncs,piped_commands,pending_results+
* %> - database server port number
* %n - database user name
* %s - service
@@ -181,7 +184,16 @@ get_prompt(promptStatus_t status, ConditionalStack cstack)
snprintf(buf, sizeof(buf), "%d", pid);
}
break;
+ case 'P':
+ {
+ PGpipelineStatus status = PQpipelineStatus(pset.db);
+ if (status == PQ_PIPELINE_ON)
+ snprintf(buf, sizeof(buf), "|%d,%d,%d|", pset.piped_syncs, pset.piped_commands, pset.available_results);
+ else if (status == PQ_PIPELINE_ABORTED)
+ snprintf(buf, sizeof(buf), "+%d,%d,%d+", pset.piped_syncs, pset.piped_commands, pset.available_results);
+ break;
+ }
case '0':
case '1':
case '2':
diff --git a/src/bin/psql/settings.h b/src/bin/psql/settings.h
index 2a8fe12eb55..89219e159e3 100644
--- a/src/bin/psql/settings.h
+++ b/src/bin/psql/settings.h
@@ -23,8 +23,8 @@
#define DEFAULT_EDITOR_LINENUMBER_ARG "+"
#endif
-#define DEFAULT_PROMPT1 "%/%R%x%# "
-#define DEFAULT_PROMPT2 "%/%R%x%# "
+#define DEFAULT_PROMPT1 "%/%R%P%x%# "
+#define DEFAULT_PROMPT2 "%/%R%P%x%# "
#define DEFAULT_PROMPT3 ">> "
/*
@@ -69,6 +69,12 @@ typedef enum
PSQL_SEND_EXTENDED_PARSE,
PSQL_SEND_EXTENDED_QUERY_PARAMS,
PSQL_SEND_EXTENDED_QUERY_PREPARED,
+ PSQL_SEND_PIPELINE_SYNC,
+ PSQL_START_PIPELINE_MODE,
+ PSQL_END_PIPELINE_MODE,
+ PSQL_FLUSH,
+ PSQL_SEND_FLUSH_REQUEST,
+ PSQL_GET_RESULTS,
} PSQL_SEND_MODE;
typedef enum
@@ -111,6 +117,12 @@ typedef struct _psqlSettings
char **bind_params; /* parameters for extended query protocol call */
char *stmtName; /* prepared statement name used for extended
* query protocol commands */
+ int piped_commands; /* number of piped commands */
+ int piped_syncs; /* number of piped syncs */
+ int available_results; /* number of results available to get */
+ int requested_results; /* number of requested results, include
+ * sync messages. Used to read a limited
+ * subset of the available_results */
bool crosstab_flag; /* one-shot request to crosstab result */
char *ctv_args[4]; /* \crosstabview arguments */
diff --git a/src/bin/psql/tab-complete.in.c b/src/bin/psql/tab-complete.in.c
index 81cbf10aa28..bd3a78d909d 100644
--- a/src/bin/psql/tab-complete.in.c
+++ b/src/bin/psql/tab-complete.in.c
@@ -1867,9 +1867,9 @@ psql_completion(const char *text, int start, int end)
"\\drds", "\\drg", "\\dRs", "\\dRp", "\\ds",
"\\dt", "\\dT", "\\dv", "\\du", "\\dx", "\\dX", "\\dy",
"\\echo", "\\edit", "\\ef", "\\elif", "\\else", "\\encoding",
- "\\endif", "\\errverbose", "\\ev",
- "\\f",
- "\\g", "\\gdesc", "\\getenv", "\\gexec", "\\gset", "\\gx",
+ "\\endif", "\\endpipeline", "\\errverbose", "\\ev",
+ "\\f", "\\flush", "\\flushrequest",
+ "\\g", "\\gdesc", "\\getenv", "\\getresults", "\\gexec", "\\gset", "\\gx",
"\\help", "\\html",
"\\if", "\\include", "\\include_relative", "\\ir",
"\\list", "\\lo_import", "\\lo_export", "\\lo_list", "\\lo_unlink",
@@ -1877,7 +1877,7 @@ psql_completion(const char *text, int start, int end)
"\\parse", "\\password", "\\print", "\\prompt", "\\pset",
"\\qecho", "\\quit",
"\\reset",
- "\\s", "\\set", "\\setenv", "\\sf", "\\sv",
+ "\\s", "\\set", "\\setenv", "\\sf", "\\startpipeline", "\\sv", "\\syncpipeline",
"\\t", "\\T", "\\timing",
"\\unset",
"\\x",
diff --git a/src/test/regress/expected/psql.out b/src/test/regress/expected/psql.out
index 36dc31c16c4..ded510d50bc 100644
--- a/src/test/regress/expected/psql.out
+++ b/src/test/regress/expected/psql.out
@@ -6828,3 +6828,671 @@ CREATE TABLE defprivs (a int);
\pset null ''
DROP TABLE defprivs;
+-- pipelining
+CREATE TABLE psql_pipeline(a INTEGER PRIMARY KEY, s TEXT);
+-- single query
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+-- multiple queries
+\startpipeline
+SELECT $1 \bind 'val1' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val2 | val3
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val2 | val3
+(1 row)
+
+-- Test \flush
+\startpipeline
+\flush
+SELECT $1 \bind 'val1' \g
+\flush
+SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val2 | val3
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val2 | val3
+(1 row)
+
+-- send multiple syncs
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\syncpipeline
+\syncpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\syncpipeline
+SELECT $1, $2 \bind 'val4' 'val5' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val2 | val3
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val4 | val5
+(1 row)
+
+-- startpipeline shouldn't have any effect if already in a pipeline
+\startpipeline
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+-- Convert an implicit tx block to an explicit tx block
+\startpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 2 \g
+ROLLBACK \bind \g
+\endpipeline
+-- Multiple explicit transactions
+\startpipeline
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+ROLLBACK \bind \g
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+COMMIT \bind \g
+\endpipeline
+-- COPY FROM STDIN
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+-- COPY FROM STDIN with \flushrequest + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\flushrequest
+\getresults
+ ?column?
+----------
+ val1
+(1 row)
+
+message type 0x5a arrived from server while idle
+\endpipeline
+-- COPY FROM STDIN with \syncpipeline + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\syncpipeline
+\getresults
+ ?column?
+----------
+ val1
+(1 row)
+
+\endpipeline
+-- COPY TO STDOUT
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+1 \N
+2 test2
+3 test3
+4 test4
+-- COPY TO STDOUT with \flushrequest + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\flushrequest
+\getresults
+ ?column?
+----------
+ val1
+(1 row)
+
+1 \N
+2 test2
+3 test3
+4 test4
+\endpipeline
+-- COPY TO STDOUT with \syncpipeline + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\syncpipeline
+\getresults
+ ?column?
+----------
+ val1
+(1 row)
+
+1 \N
+2 test2
+3 test3
+4 test4
+\endpipeline
+-- Use \parse and \bind_named
+\startpipeline
+SELECT $1 \parse ''
+SELECT $1, $2 \parse ''
+SELECT $2 \parse pipeline_1
+\bind_named '' 1 2 \g
+\bind_named pipeline_1 2 \g
+\endpipeline
+ERROR: could not determine data type of parameter $1
+-- \getresults displays all results preceding a \flushrequest
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+\flushrequest
+\getresults
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+\endpipeline
+-- \getresults displays all results preceding a sync
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+\syncpipeline
+\getresults
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+\endpipeline
+-- \getresults immediately returns if there's no result to fetch
+\startpipeline
+\getresults
+No pending results to get
+SELECT $1 \bind 2 \g
+\getresults
+No pending results to get
+\flushrequest
+\endpipeline
+ ?column?
+----------
+ 2
+(1 row)
+
+\getresults
+No pending results to get
+-- \getresults only fetch results preceding a flushrequest
+\startpipeline
+SELECT $1 \bind 2 \g
+\flushrequest
+SELECT $1 \bind 2 \g
+\getresults
+ ?column?
+----------
+ 2
+(1 row)
+
+\endpipeline
+ ?column?
+----------
+ 2
+(1 row)
+
+-- \getresults only fetch results preceding a sync message
+\startpipeline
+SELECT $1 \bind 2 \g
+\syncpipeline
+SELECT $1 \bind 2 \g
+\getresults
+ ?column?
+----------
+ 2
+(1 row)
+
+\endpipeline
+ ?column?
+----------
+ 2
+(1 row)
+
+-- use pipeline with chunked results with both \getresults and \endpipeline
+\startpipeline
+\set FETCH_COUNT 10
+SELECT $1 \bind 2 \g
+\flushrequest
+\getresults
+ ?column?
+----------
+ 2
+(1 row)
+
+SELECT $1 \bind 2 \g
+\endpipeline
+ ?column?
+----------
+ 2
+(1 row)
+
+\unset FETCH_COUNT
+-- \getresults with specific number of requested results
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+SELECT $1 \bind 3 \g
+\syncpipeline
+\getresults 1
+ ?column?
+----------
+ 1
+(1 row)
+
+SELECT $1 \bind 4 \g
+\getresults 3
+ ?column?
+----------
+ 2
+(1 row)
+
+ ?column?
+----------
+ 3
+(1 row)
+
+\endpipeline
+ ?column?
+----------
+ 4
+(1 row)
+
+-- \syncpipeline count as one command to fetch for \getresults
+\startpipeline
+\syncpipeline
+\syncpipeline
+SELECT $1 \bind 1 \g
+\flushrequest
+\getresults 2
+\getresults 1
+ ?column?
+----------
+ 1
+(1 row)
+
+\endpipeline
+-- \getresults 0 should get all results
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+SELECT $1 \bind 3 \g
+\syncpipeline
+\getresults 0
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+ ?column?
+----------
+ 3
+(1 row)
+
+\endpipeline
+-- pipelining errors
+-- endpipeline outside of pipeline should fail
+\endpipeline
+cannot send pipeline when not in pipeline mode
+-- Query using simple protocol should not be sent and should leave the pipeline usable
+\startpipeline
+SELECT 1;
+PQsendQuery not allowed in pipeline mode
+SELECT $1 \bind 'val1' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+-- After an aborted pipeline, commands after a sync should be displayed
+\startpipeline
+SELECT $1 \bind \g
+\syncpipeline
+SELECT $1 \bind 1 \g
+\endpipeline
+ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
+ ?column?
+----------
+ 1
+(1 row)
+
+-- Incorrect number of parameters, the pipeline will be aborted and following queries won't be executed
+\startpipeline
+SELECT \bind 'val1' \g
+SELECT $1 \bind 'val1' \g
+\endpipeline
+ERROR: bind message supplies 1 parameters, but prepared statement "" requires 0
+-- An explicit transaction with an error needs to be rollbacked after the pipeline
+\startpipeline
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+ROLLBACK \bind \g
+\endpipeline
+ERROR: duplicate key value violates unique constraint "psql_pipeline_pkey"
+DETAIL: Key (a)=(1) already exists.
+ROLLBACK;
+-- \watch sends a simple query which won't be allowed within a pipeline
+\startpipeline
+SELECT \bind \g
+\watch 1
+PQsendQuery not allowed in pipeline mode
+
+\endpipeline
+--
+(1 row)
+
+-- \gdesc should fail as synchronous commands are not allowed in pipeline, pipeline should still be usable
+\startpipeline
+SELECT $1 \bind 1 \gdesc
+synchronous command execution functions are not allowed in pipeline mode
+SELECT $1 \bind 1 \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+-- \gset is not allowed, pipeline should still be usable
+\startpipeline
+SELECT $1 as i, $2 as j \parse ''
+SELECT $1 as k, $2 as l \parse 'second'
+\bind_named '' 1 2 \gset
+\gset not allowed in pipeline mode
+\bind_named second 1 2 \gset pref02_ \echo :pref02_i :pref02_j
+\gset not allowed in pipeline mode
+\bind_named '' 1 2 \g
+\endpipeline
+ i | j
+---+---
+ 1 | 2
+(1 row)
+
+-- \gx is not allowed, pipeline should still be usable
+\startpipeline
+SELECT $1 \bind 1 \gx
+\gx not allowed in pipeline mode
+\reset
+SELECT $1 \bind 1 \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+-- \gx warning should be emitted in an aborted pipeline
+\startpipeline
+SELECT $1 \bind \g
+\flushrequest
+\getresults
+ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
+SELECT $1 \bind 1 \gx
+\gx not allowed in pipeline mode
+\endpipeline
+-- \gexec is not allowed, pipeline should still be usable
+\startpipeline
+SELECT 'INSERT INTO psql_pipeline(a) SELECT generate_series(1, 10)' \parse 'insert_stmt'
+\bind_named insert_stmt \gexec
+\gexec not allowed in pipeline mode
+\bind_named insert_stmt \g
+SELECT COUNT(*) FROM psql_pipeline \bind \g
+\endpipeline
+ ?column?
+------------------------------------------------------------
+ INSERT INTO psql_pipeline(a) SELECT generate_series(1, 10)
+(1 row)
+
+ count
+-------
+ 4
+(1 row)
+
+-- After an error, pipeline is aborted and requires a sync to be reusable
+\startpipeline
+SELECT $1 \bind \g
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+\flushrequest
+\getresults
+ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
+-- Pipeline is aborted
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+-- Sync allows pipeline to recover
+\syncpipeline
+\getresults
+Pipeline aborted, command didn't run
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+\flushrequest
+\getresults
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 1
+(1 row)
+
+\endpipeline
+-- In an aborted pipeline, \getresults 1 aborted commands one at a time
+\startpipeline
+SELECT $1 \bind \g
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\syncpipeline
+\getresults 1
+ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
+\getresults 1
+Pipeline aborted, command didn't run
+\getresults 1
+Pipeline aborted, command didn't run
+\getresults 1
+Pipeline aborted, command didn't run
+\getresults 1
+\endpipeline
+-- Test chunked results with an aborted pipeline
+\startpipeline
+\set FETCH_COUNT 10
+SELECT $1 \bind \g
+\flushrequest
+\getresults
+ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
+SELECT $1 \bind \g
+\endpipeline
+fetching results in chunked mode failed
+Pipeline aborted, command didn't run
+\unset FETCH_COUNT
+-- \getresults returns an error when an incorrect number is provided
+\startpipeline
+\getresults -1
+\getresults: invalid number of requested results
+\endpipeline
+-- \getresults when there's no result shouldn't impact the following query
+\getresults 1
+No pending results to get
+select 1;
+ ?column?
+----------
+ 1
+(1 row)
+
+-- pipelining and transaction block behaviour
+-- set local will issue a warning when modifying a GUC outside of a transaction block
+-- The change will still be valid as a pipeline runs within an implicit transaction block
+-- Sending a sync will commit the implicit transaction block. The first command after a sync
+-- won't be seen as belonging to a pipeline.
+\startpipeline
+SET LOCAL statement_timeout='1h' \bind \g
+SHOW statement_timeout \bind \g
+\syncpipeline
+SHOW statement_timeout \bind \g
+SET LOCAL statement_timeout='2h' \bind \g
+SHOW statement_timeout \bind \g
+\endpipeline
+WARNING: SET LOCAL can only be used in transaction blocks
+ statement_timeout
+-------------------
+ 1h
+(1 row)
+
+ statement_timeout
+-------------------
+ 0
+(1 row)
+
+ statement_timeout
+-------------------
+ 2h
+(1 row)
+
+-- Reindex concurrently is forbidden in the middle of a pipeline
+\startpipeline
+SELECT $1 \bind 1 \g
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ERROR: REINDEX CONCURRENTLY cannot run inside a transaction block
+-- Reindex concurrently will work if it's the first command of a pipeline
+\startpipeline
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ ?column?
+----------
+ 2
+(1 row)
+
+-- subtransactions are not allowed in pipeline mode
+\startpipeline
+SAVEPOINT a \bind \g
+SELECT $1 \bind 1 \g
+ROLLBACK TO SAVEPOINT a \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ERROR: SAVEPOINT can only be used in transaction blocks
+-- Lock command will fail as first pipeline command is not seen as a transaction block
+\startpipeline
+LOCK psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ERROR: LOCK TABLE can only be used in transaction blocks
+-- Lock command will succeed after the first command as pipeline will be seen as an implicit transaction block
+\startpipeline
+SELECT $1 \bind 1 \g
+LOCK psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+-- Vacuum command will work as the first command
+\startpipeline
+VACUUM psql_pipeline \bind \g
+\endpipeline
+-- Vacuum command will fail within pipeline implicit transaction
+\startpipeline
+SELECT 1 \bind \g
+VACUUM psql_pipeline \bind \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ERROR: VACUUM cannot run inside a transaction block
+-- Vacuum command will work after a sync
+\startpipeline
+SELECT 1 \bind \g
+\syncpipeline
+VACUUM psql_pipeline \bind \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
diff --git a/src/test/regress/sql/psql.sql b/src/test/regress/sql/psql.sql
index c5021fc0b13..33b204edd7f 100644
--- a/src/test/regress/sql/psql.sql
+++ b/src/test/regress/sql/psql.sql
@@ -1929,3 +1929,402 @@ CREATE TABLE defprivs (a int);
\z defprivs
\pset null ''
DROP TABLE defprivs;
+
+-- pipelining
+CREATE TABLE psql_pipeline(a INTEGER PRIMARY KEY, s TEXT);
+
+-- single query
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\endpipeline
+
+-- multiple queries
+\startpipeline
+SELECT $1 \bind 'val1' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\endpipeline
+
+-- Test \flush
+\startpipeline
+\flush
+SELECT $1 \bind 'val1' \g
+\flush
+SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\endpipeline
+
+-- send multiple syncs
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\syncpipeline
+\syncpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\syncpipeline
+SELECT $1, $2 \bind 'val4' 'val5' \g
+\endpipeline
+
+-- startpipeline shouldn't have any effect if already in a pipeline
+\startpipeline
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\endpipeline
+
+-- Convert an implicit tx block to an explicit tx block
+\startpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 2 \g
+ROLLBACK \bind \g
+\endpipeline
+
+-- Multiple explicit transactions
+\startpipeline
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+ROLLBACK \bind \g
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+COMMIT \bind \g
+\endpipeline
+
+-- COPY FROM STDIN
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\endpipeline
+2 test2
+\.
+
+-- COPY FROM STDIN with \flushrequest + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\flushrequest
+\getresults
+3 test3
+\.
+\endpipeline
+
+-- COPY FROM STDIN with \syncpipeline + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\syncpipeline
+\getresults
+4 test4
+\.
+\endpipeline
+
+-- COPY TO STDOUT
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\endpipeline
+
+-- COPY TO STDOUT with \flushrequest + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\flushrequest
+\getresults
+\endpipeline
+
+-- COPY TO STDOUT with \syncpipeline + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\syncpipeline
+\getresults
+\endpipeline
+
+-- Use \parse and \bind_named
+\startpipeline
+SELECT $1 \parse ''
+SELECT $1, $2 \parse ''
+SELECT $2 \parse pipeline_1
+\bind_named '' 1 2 \g
+\bind_named pipeline_1 2 \g
+\endpipeline
+
+-- \getresults displays all results preceding a \flushrequest
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+\flushrequest
+\getresults
+\endpipeline
+
+-- \getresults displays all results preceding a sync
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+\syncpipeline
+\getresults
+\endpipeline
+
+-- \getresults immediately returns if there's no result to fetch
+\startpipeline
+\getresults
+SELECT $1 \bind 2 \g
+\getresults
+\flushrequest
+\endpipeline
+\getresults
+
+-- \getresults only fetch results preceding a flushrequest
+\startpipeline
+SELECT $1 \bind 2 \g
+\flushrequest
+SELECT $1 \bind 2 \g
+\getresults
+\endpipeline
+
+-- \getresults only fetch results preceding a sync message
+\startpipeline
+SELECT $1 \bind 2 \g
+\syncpipeline
+SELECT $1 \bind 2 \g
+\getresults
+\endpipeline
+
+-- use pipeline with chunked results with both \getresults and \endpipeline
+\startpipeline
+\set FETCH_COUNT 10
+SELECT $1 \bind 2 \g
+\flushrequest
+\getresults
+SELECT $1 \bind 2 \g
+\endpipeline
+\unset FETCH_COUNT
+
+-- \getresults with specific number of requested results
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+SELECT $1 \bind 3 \g
+\syncpipeline
+\getresults 1
+SELECT $1 \bind 4 \g
+\getresults 3
+\endpipeline
+
+-- \syncpipeline count as one command to fetch for \getresults
+\startpipeline
+\syncpipeline
+\syncpipeline
+SELECT $1 \bind 1 \g
+\flushrequest
+\getresults 2
+\getresults 1
+\endpipeline
+
+-- \getresults 0 should get all results
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+SELECT $1 \bind 3 \g
+\syncpipeline
+\getresults 0
+\endpipeline
+
+-- pipelining errors
+
+-- endpipeline outside of pipeline should fail
+\endpipeline
+
+-- Query using simple protocol should not be sent and should leave the pipeline usable
+\startpipeline
+SELECT 1;
+SELECT $1 \bind 'val1' \g
+\endpipeline
+
+-- After an aborted pipeline, commands after a sync should be displayed
+\startpipeline
+SELECT $1 \bind \g
+\syncpipeline
+SELECT $1 \bind 1 \g
+\endpipeline
+
+-- Incorrect number of parameters, the pipeline will be aborted and following queries won't be executed
+\startpipeline
+SELECT \bind 'val1' \g
+SELECT $1 \bind 'val1' \g
+\endpipeline
+
+-- An explicit transaction with an error needs to be rollbacked after the pipeline
+\startpipeline
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+ROLLBACK \bind \g
+\endpipeline
+ROLLBACK;
+
+-- \watch sends a simple query which won't be allowed within a pipeline
+\startpipeline
+SELECT \bind \g
+\watch 1
+\endpipeline
+
+-- \gdesc should fail as synchronous commands are not allowed in pipeline, pipeline should still be usable
+\startpipeline
+SELECT $1 \bind 1 \gdesc
+SELECT $1 \bind 1 \g
+\endpipeline
+
+-- \gset is not allowed, pipeline should still be usable
+\startpipeline
+SELECT $1 as i, $2 as j \parse ''
+SELECT $1 as k, $2 as l \parse 'second'
+\bind_named '' 1 2 \gset
+\bind_named second 1 2 \gset pref02_ \echo :pref02_i :pref02_j
+\bind_named '' 1 2 \g
+\endpipeline
+
+-- \gx is not allowed, pipeline should still be usable
+\startpipeline
+SELECT $1 \bind 1 \gx
+\reset
+SELECT $1 \bind 1 \g
+\endpipeline
+
+-- \gx warning should be emitted in an aborted pipeline
+\startpipeline
+SELECT $1 \bind \g
+\flushrequest
+\getresults
+SELECT $1 \bind 1 \gx
+\endpipeline
+
+-- \gexec is not allowed, pipeline should still be usable
+\startpipeline
+SELECT 'INSERT INTO psql_pipeline(a) SELECT generate_series(1, 10)' \parse 'insert_stmt'
+\bind_named insert_stmt \gexec
+\bind_named insert_stmt \g
+SELECT COUNT(*) FROM psql_pipeline \bind \g
+\endpipeline
+
+-- After an error, pipeline is aborted and requires a sync to be reusable
+\startpipeline
+SELECT $1 \bind \g
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+\flushrequest
+\getresults
+-- Pipeline is aborted
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+-- Sync allows pipeline to recover
+\syncpipeline
+\getresults
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+\flushrequest
+\getresults
+\endpipeline
+
+-- In an aborted pipeline, \getresults 1 aborted commands one at a time
+\startpipeline
+SELECT $1 \bind \g
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\syncpipeline
+\getresults 1
+\getresults 1
+\getresults 1
+\getresults 1
+\getresults 1
+\endpipeline
+
+-- Test chunked results with an aborted pipeline
+\startpipeline
+\set FETCH_COUNT 10
+SELECT $1 \bind \g
+\flushrequest
+\getresults
+SELECT $1 \bind \g
+\endpipeline
+\unset FETCH_COUNT
+
+-- \getresults returns an error when an incorrect number is provided
+\startpipeline
+\getresults -1
+\endpipeline
+
+-- \getresults when there's no result shouldn't impact the following query
+\getresults 1
+select 1;
+
+-- pipelining and transaction block behaviour
+
+-- set local will issue a warning when modifying a GUC outside of a transaction block
+-- The change will still be valid as a pipeline runs within an implicit transaction block
+-- Sending a sync will commit the implicit transaction block. The first command after a sync
+-- won't be seen as belonging to a pipeline.
+\startpipeline
+SET LOCAL statement_timeout='1h' \bind \g
+SHOW statement_timeout \bind \g
+\syncpipeline
+SHOW statement_timeout \bind \g
+SET LOCAL statement_timeout='2h' \bind \g
+SHOW statement_timeout \bind \g
+\endpipeline
+
+-- Reindex concurrently is forbidden in the middle of a pipeline
+\startpipeline
+SELECT $1 \bind 1 \g
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- Reindex concurrently will work if it's the first command of a pipeline
+\startpipeline
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- subtransactions are not allowed in pipeline mode
+\startpipeline
+SAVEPOINT a \bind \g
+SELECT $1 \bind 1 \g
+ROLLBACK TO SAVEPOINT a \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- Lock command will fail as first pipeline command is not seen as a transaction block
+\startpipeline
+LOCK psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- Lock command will succeed after the first command as pipeline will be seen as an implicit transaction block
+\startpipeline
+SELECT $1 \bind 1 \g
+LOCK psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- Vacuum command will work as the first command
+\startpipeline
+VACUUM psql_pipeline \bind \g
+\endpipeline
+
+-- Vacuum command will fail within pipeline implicit transaction
+\startpipeline
+SELECT 1 \bind \g
+VACUUM psql_pipeline \bind \g
+\endpipeline
+
+-- Vacuum command will work after a sync
+\startpipeline
+SELECT 1 \bind \g
+\syncpipeline
+VACUUM psql_pipeline \bind \g
+\endpipeline
--
2.39.5 (Apple Git-154)
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-02-18 07:22 Michael Paquier <[email protected]>
parent: Anthonin Bonnefoy <[email protected]>
0 siblings, 1 reply; 41+ messages in thread
From: Michael Paquier @ 2025-02-18 07:22 UTC (permalink / raw)
To: Anthonin Bonnefoy <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; pgsql-hackers
On Tue, Jan 14, 2025 at 09:49:02AM +0100, Anthonin Bonnefoy wrote:
> During my tests, I've noticed I didn't handle query cancellation, this
> is now fixed. I've also added additional comments related to
> available_results to make it clearer that it depends on what the
> server has flushed to the client.
This is a pretty cool patch. I like the structure you have used for
the integration with the tracking of the number of commands, the
number of syncs (like pgbench) put in a pipeline, the number of
results requested and the number of results available. That makes the
whole easier to look at with a state in pset.
+ PSQL_SEND_PIPELINE_SYNC,
+ PSQL_START_PIPELINE_MODE,
+ PSQL_END_PIPELINE_MODE,
+ PSQL_FLUSH,
+ PSQL_SEND_FLUSH_REQUEST,
+ PSQL_GET_RESULTS,
These new values are inconsistent, let's use some more PSQL_SEND_*
here. That makes the whole set of values more greppable.
The tests in psql.sql are becoming really long. Perhaps it would be
better to split that into its own file, say psql_pipeline.sql? The
input file is already 2k lines, you are adding 15% more lines to that.
+ * otherwise, calling PQgetResults will block.
Likely PQgetResults => PQgetResult().
Wondering if the cancellation in ExecQueryAndProcessResults() is
sound, I've not been able to break it, still..
I can also get behind the additions of \flush and \flushrequest to
query different parts of libpq.
+ if (pset.requested_results == 0)
+ /* We've read all requested results, exit */
+ return res;
Set of nits with the style of the code, but I'd suggest to use
braces {} here, to outline that the comment is in a block. There's a
second, larger one in discardAbortedPipelineResults().
+ if (strcmp(cmd, "gx") == 0 && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pg_log_error("\\gx not allowed in pipeline mode");
+ clean_extended_state();
+ return PSQL_CMD_ERROR;
+ }
What is the reasoning here behind this restriction? \gx is a wrapper
of \g with expanded mode on, but it is also possible to call \g with
expanded=on, bypassing this restriction.
Let's split the prompt patch with the support of %P into its own
patch.
-#define DEFAULT_PROMPT1 "%/%R%x%# "
-#define DEFAULT_PROMPT2 "%/%R%x%# "
+#define DEFAULT_PROMPT1 "%/%R%P%x%# "
+#define DEFAULT_PROMPT2 "%/%R%P%x%# "
#define DEFAULT_PROMPT3 ">> "
I don't think that changing this default is a good idea. Everybody
can do that in their own .psqlrc (spoiler: I do).
The idea to use three fields with a hardcoded format does not look
like a good idea to me. I think that this should be done in a
different and more extensible way:
- Use %P to track if we are in pipeline mode on, off or abort.
- Define three new variables that behave like ROW_COUNT, but for the
fields you want to track here. These could then be passed down to a
PROMPT with %:name:, grammar already supported.
That would make the whole much more flexible. At it seems to me that
we could also add requested_results to this set? These could be named
with the same prefix, like PIPELINE_SYNC_COUNT, etc.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-02-18 17:34 Anthonin Bonnefoy <[email protected]>
parent: Michael Paquier <[email protected]>
0 siblings, 2 replies; 41+ messages in thread
From: Anthonin Bonnefoy @ 2025-02-18 17:34 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; pgsql-hackers
On Tue, Feb 18, 2025 at 8:23 AM Michael Paquier <[email protected]> wrote:
> This is a pretty cool patch. I like the structure you have used for
> the integration with the tracking of the number of commands, the
> number of syncs (like pgbench) put in a pipeline, the number of
> results requested and the number of results available. That makes the
> whole easier to look at with a state in pset.
Thanks!
> + PSQL_SEND_PIPELINE_SYNC,
> + PSQL_START_PIPELINE_MODE,
> + PSQL_END_PIPELINE_MODE,
> + PSQL_FLUSH,
> + PSQL_SEND_FLUSH_REQUEST,
> + PSQL_GET_RESULTS,
>
> These new values are inconsistent, let's use some more PSQL_SEND_*
> here. That makes the whole set of values more greppable.
Changed.
> The tests in psql.sql are becoming really long. Perhaps it would be
> better to split that into its own file, say psql_pipeline.sql? The
> input file is already 2k lines, you are adding 15% more lines to that.
Agreed, I wasn't sure if this was enough to warrant a dedicated test
file. This is now separated in psql_pipeline.sql.
> + * otherwise, calling PQgetResults will block.
>
> Likely PQgetResults => PQgetResult().
Indeed, this is fixed.
> Set of nits with the style of the code, but I'd suggest to use
> braces {} here, to outline that the comment is in a block. There's a
> second, larger one in discardAbortedPipelineResults().
>
> + if (strcmp(cmd, "gx") == 0 && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
> + {
> + pg_log_error("\\gx not allowed in pipeline mode");
> + clean_extended_state();
> + return PSQL_CMD_ERROR;
> + }
Changed.
> What is the reasoning here behind this restriction? \gx is a wrapper
> of \g with expanded mode on, but it is also possible to call \g with
> expanded=on, bypassing this restriction.
The issue is that \gx enables expanded mode for the duration of the
query and immediately reset it in sendquery_cleanup. With pipelining,
the command is piped and displaying is done by either \endpipeline or
\getresults, so the flag change has no impact. Forbidding it was a way
to make it clearer that it won't have the expected effect. If we
wanted a similar feature, this would need to be done with something
like \endpipelinex or \getresultsx.
> Let's split the prompt patch with the support of %P into its own
> patch.
>
> -#define DEFAULT_PROMPT1 "%/%R%x%# "
> -#define DEFAULT_PROMPT2 "%/%R%x%# "
> +#define DEFAULT_PROMPT1 "%/%R%P%x%# "
> +#define DEFAULT_PROMPT2 "%/%R%P%x%# "
> #define DEFAULT_PROMPT3 ">> "
>
> I don't think that changing this default is a good idea. Everybody
> can do that in their own .psqlrc (spoiler: I do).
>
> The idea to use three fields with a hardcoded format does not look
> like a good idea to me. I think that this should be done in a
> different and more extensible way:
> - Use %P to track if we are in pipeline mode on, off or abort.
> - Define three new variables that behave like ROW_COUNT, but for the
> fields you want to track here. These could then be passed down to a
> PROMPT with %:name:, grammar already supported.
>
> That would make the whole much more flexible. At it seems to me that
> we could also add requested_results to this set? These could be named
> with the same prefix, like PIPELINE_SYNC_COUNT, etc.
I've split the patch and created the 3 special variables:
PIPELINE_SYNC_COUNT, PIPELINE_COMMAND_COUNT, PIPELINE_RESULT_COUNT.
For requested_results, I don't think there's value in exposing it
since it is used as an exit condition and thus will always be 0
outside of ExecQueryAndProcessResults.
Attachments:
[application/octet-stream] v08-0001-Add-pipelining-support-in-psql.patch (40.3K, ../../CAO6_XqrrjJRhKNhYa9Cg=_BSH8MmS1tAHMFq-H7fH=NcWJ-jvw@mail.gmail.com/2-v08-0001-Add-pipelining-support-in-psql.patch)
download | inline diff:
From a9f60a6f01922f147a869518bcacfc9ad1e88ed9 Mon Sep 17 00:00:00 2001
From: Anthonin Bonnefoy <[email protected]>
Date: Tue, 5 Nov 2024 10:26:54 +0100
Subject: Add pipelining support in psql
With \bind, \parse, \bind_named and \close, it is possible to issue
queries from psql using the extended protocol. However, it wasn't
possible to send those queries using pipelining and the only way to test
pipelined queries was through pgbench's tap tests.
This patch adds additional psql meta-commands to support pipelining:
\startpipeline, \endpipeline and \syncpipeline, mirroring the existing
meta-commands in pgbench. Additional meta-commands allow to flush and
read results of an ongoing pipeline: \flushrequest, \flush and \getresults
\startpipeline starts a new pipeline. All extended queries will be
queued until the end of the pipeline is reached.
\endpipeline ends an ongoing pipeline. All queued commands will be sent
to the server and all responses will be processed by the psql.
\syncpipeline queues a synchronisation point without flushing the
commands to the server
\flush Call PQflush on psql's connection
\flushrequest queues a flushrequest
\getresults reads server's results. Unsent data are automatically pushed
when \getresults is called
Those meta-commands will allow to test pipeline behaviour using
psql regression tests.
---
doc/src/sgml/ref/psql-ref.sgml | 70 +++++
src/bin/psql/command.c | 151 ++++++++++
src/bin/psql/common.c | 279 ++++++++++++++++-
src/bin/psql/help.c | 7 +
src/bin/psql/settings.h | 12 +
src/bin/psql/tab-complete.in.c | 8 +-
src/test/regress/parallel_schedule | 2 +-
src/test/regress/sql/psql_pipeline.sql | 401 +++++++++++++++++++++++++
8 files changed, 919 insertions(+), 11 deletions(-)
create mode 100644 src/test/regress/sql/psql_pipeline.sql
diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml
index f3044fac1fa..fdae1ed0479 100644
--- a/doc/src/sgml/ref/psql-ref.sgml
+++ b/doc/src/sgml/ref/psql-ref.sgml
@@ -3674,6 +3674,76 @@ testdb=> <userinput>\setenv LESS -imx4F</userinput>
</listitem>
</varlistentry>
+ <varlistentry id="app-psql-meta-command-pipeline">
+ <term><literal>\startpipeline</literal></term>
+ <term><literal>\syncpipeline</literal></term>
+ <term><literal>\endpipeline</literal></term>
+ <term><literal>\flushrequest</literal></term>
+ <term><literal>\flush</literal></term>
+ <term><literal>\getresults [ <replaceable class="parameter">number_results</replaceable> ]</literal></term>
+
+ <listitem>
+ <para>
+ This group of commands implements pipelining of SQL statements.
+ A pipeline must begin with a <command>\startpipeline</command>
+ and end with an <command>\endpipeline</command>. In between there
+ may be any number of <command>\syncpipeline</command> commands,
+ which sends a <link linkend="protocol-flow-ext-query">sync message</link>
+ without ending the ongoing pipeline and flushing the send buffer.
+ In pipeline mode, statements are sent to the server without waiting
+ for the results of previous statements. See
+ <xref linkend="libpq-pipeline-mode"/> for more details.
+ </para>
+
+ <para>
+ Pipeline mode requires the use of extended query protocol. All queries need
+ to be sent using the meta-commands <literal>\bind</literal>,
+ <literal>\bind_named</literal>, <literal>\close</literal> or
+ <literal>\parse</literal>. While a pipeline is ongoing,
+ <literal>\g</literal> will append the current query buffer to the pipeline and
+ other meta-commands like <literal>\gx</literal> or <literal>\gdesc</literal>
+ are not allowed in pipeline mode.
+ </para>
+
+ <para>
+ <command>\flushrequest</command> appends a flush command to the pipeline,
+ allowing to read results with <command>\getresults</command> without issuing
+ a sync or ending the pipeline. <command>\getresults</command> will automatically
+ push unsent data to the server. <command>\flush</command> can be used to manually
+ push unsent data.
+ </para>
+
+ <para>
+ <command>\getresults</command> accepts an optional
+ <replaceable class="parameter">number_results</replaceable> parameter. If provided,
+ only the first <replaceable class="parameter">number_results</replaceable> pending
+ results will be read. If not provided or 0, all pending results are read. The
+ commands <literal>\bind</literal>, <literal>\bind_named</literal>,
+ <literal>\close</literal>, <literal>\parse</literal> and <literal>\syncpipeline</literal>
+ generate one result to get.
+ </para>
+
+ <para>
+ Example:
+<programlisting>
+\startpipeline
+SELECT 1 \bind \g
+SELECT $1 \parse stmt1
+\bind_named stmt1 1 \g
+SELECT pg_current_xact_id() \bind \g
+\flushrequest
+\getresults
+\syncpipeline
+SELECT pg_current_xact_id() \bind \g
+\flushrequest
+\getresults 2
+\close stmt1
+\endpipeline
+</programlisting></para>
+
+ </listitem>
+ </varlistentry>
+
<varlistentry id="app-psql-meta-command-t-lc">
<term><literal>\t</literal></term>
diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c
index 26dfdde195a..09b3afc634c 100644
--- a/src/bin/psql/command.c
+++ b/src/bin/psql/command.c
@@ -90,9 +90,12 @@ static backslashResult exec_command_else(PsqlScanState scan_state, ConditionalSt
PQExpBuffer query_buf);
static backslashResult exec_command_endif(PsqlScanState scan_state, ConditionalStack cstack,
PQExpBuffer query_buf);
+static backslashResult exec_command_endpipeline(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_encoding(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_errverbose(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_f(PsqlScanState scan_state, bool active_branch);
+static backslashResult exec_command_flush(PsqlScanState scan_state, bool active_branch);
+static backslashResult exec_command_flushrequest(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_g(PsqlScanState scan_state, bool active_branch,
const char *cmd);
static backslashResult process_command_g_options(char *first_option,
@@ -103,6 +106,7 @@ static backslashResult exec_command_gdesc(PsqlScanState scan_state, bool active_
static backslashResult exec_command_getenv(PsqlScanState scan_state, bool active_branch,
const char *cmd);
static backslashResult exec_command_gexec(PsqlScanState scan_state, bool active_branch);
+static backslashResult exec_command_getresults(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_gset(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_help(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_html(PsqlScanState scan_state, bool active_branch);
@@ -132,6 +136,8 @@ static backslashResult exec_command_setenv(PsqlScanState scan_state, bool active
const char *cmd);
static backslashResult exec_command_sf_sv(PsqlScanState scan_state, bool active_branch,
const char *cmd, bool is_func);
+static backslashResult exec_command_startpipeline(PsqlScanState scan_state, bool active_branch);
+static backslashResult exec_command_syncpipeline(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_t(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_T(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_timing(PsqlScanState scan_state, bool active_branch);
@@ -351,18 +357,26 @@ exec_command(const char *cmd,
status = exec_command_else(scan_state, cstack, query_buf);
else if (strcmp(cmd, "endif") == 0)
status = exec_command_endif(scan_state, cstack, query_buf);
+ else if (strcmp(cmd, "endpipeline") == 0)
+ status = exec_command_endpipeline(scan_state, active_branch);
else if (strcmp(cmd, "encoding") == 0)
status = exec_command_encoding(scan_state, active_branch);
else if (strcmp(cmd, "errverbose") == 0)
status = exec_command_errverbose(scan_state, active_branch);
else if (strcmp(cmd, "f") == 0)
status = exec_command_f(scan_state, active_branch);
+ else if (strcmp(cmd, "flush") == 0)
+ status = exec_command_flush(scan_state, active_branch);
+ else if (strcmp(cmd, "flushrequest") == 0)
+ status = exec_command_flushrequest(scan_state, active_branch);
else if (strcmp(cmd, "g") == 0 || strcmp(cmd, "gx") == 0)
status = exec_command_g(scan_state, active_branch, cmd);
else if (strcmp(cmd, "gdesc") == 0)
status = exec_command_gdesc(scan_state, active_branch);
else if (strcmp(cmd, "getenv") == 0)
status = exec_command_getenv(scan_state, active_branch, cmd);
+ else if (strcmp(cmd, "getresults") == 0)
+ status = exec_command_getresults(scan_state, active_branch);
else if (strcmp(cmd, "gexec") == 0)
status = exec_command_gexec(scan_state, active_branch);
else if (strcmp(cmd, "gset") == 0)
@@ -411,6 +425,10 @@ exec_command(const char *cmd,
status = exec_command_sf_sv(scan_state, active_branch, cmd, true);
else if (strcmp(cmd, "sv") == 0 || strcmp(cmd, "sv+") == 0)
status = exec_command_sf_sv(scan_state, active_branch, cmd, false);
+ else if (strcmp(cmd, "startpipeline") == 0)
+ status = exec_command_startpipeline(scan_state, active_branch);
+ else if (strcmp(cmd, "syncpipeline") == 0)
+ status = exec_command_syncpipeline(scan_state, active_branch);
else if (strcmp(cmd, "t") == 0)
status = exec_command_t(scan_state, active_branch);
else if (strcmp(cmd, "T") == 0)
@@ -1515,6 +1533,38 @@ exec_command_f(PsqlScanState scan_state, bool active_branch)
return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR;
}
+/*
+ * \flush -- call PQflush on the connection
+ */
+static backslashResult
+exec_command_flush(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_SEND_FLUSH;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
+/*
+ * \flushrequest -- send a flush request to the server
+ */
+static backslashResult
+exec_command_flushrequest(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_SEND_FLUSH_REQUEST;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
/*
* \g [(pset-option[=pset-value] ...)] [filename/shell-command]
* \gx [(pset-option[=pset-value] ...)] [filename/shell-command]
@@ -1550,6 +1600,13 @@ exec_command_g(PsqlScanState scan_state, bool active_branch, const char *cmd)
if (status == PSQL_CMD_SKIP_LINE && active_branch)
{
+ if (strcmp(cmd, "gx") == 0 && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pg_log_error("\\gx not allowed in pipeline mode");
+ clean_extended_state();
+ return PSQL_CMD_ERROR;
+ }
+
if (!fname)
pset.gfname = NULL;
else
@@ -1703,6 +1760,39 @@ exec_command_getenv(PsqlScanState scan_state, bool active_branch,
return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR;
}
+/*
+ * \getresults -- read results
+ */
+static backslashResult
+exec_command_getresults(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ char *opt;
+ int num_results;
+
+ pset.send_mode = PSQL_SEND_GET_RESULTS;
+ opt = psql_scan_slash_option(scan_state, OT_NORMAL, NULL, false);
+
+ pset.requested_results = 0;
+ if (opt != NULL)
+ {
+ num_results = atoi(opt);
+ if (num_results < 0)
+ {
+ pg_log_error("\\getresults: invalid number of requested results");
+ return PSQL_CMD_SKIP_LINE;
+ }
+ pset.requested_results = num_results;
+ }
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
+
/*
* \gexec -- send query and execute each field of result
*/
@@ -1713,6 +1803,12 @@ exec_command_gexec(PsqlScanState scan_state, bool active_branch)
if (active_branch)
{
+ if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pg_log_error("\\gexec not allowed in pipeline mode");
+ clean_extended_state();
+ return PSQL_CMD_ERROR;
+ }
pset.gexec_flag = true;
status = PSQL_CMD_SEND;
}
@@ -1733,6 +1829,13 @@ exec_command_gset(PsqlScanState scan_state, bool active_branch)
char *prefix = psql_scan_slash_option(scan_state,
OT_NORMAL, NULL, false);
+ if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pg_log_error("\\gset not allowed in pipeline mode");
+ clean_extended_state();
+ return PSQL_CMD_ERROR;
+ }
+
if (prefix)
pset.gset_prefix = prefix;
else
@@ -2718,6 +2821,54 @@ exec_command_sf_sv(PsqlScanState scan_state, bool active_branch,
return status;
}
+/*
+ * \startpipeline -- enter pipeline mode
+ */
+static backslashResult
+exec_command_startpipeline(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_SEND_START_PIPELINE_MODE;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
+/*
+ * \syncpipeline -- send a sync message to an active pipeline
+ */
+static backslashResult
+exec_command_syncpipeline(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_SEND_PIPELINE_SYNC;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
+/*
+ * \endpipeline -- end pipeline mode
+ */
+static backslashResult
+exec_command_endpipeline(PsqlScanState scan_state, bool active_branch)
+{
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_SEND_END_PIPELINE_MODE;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return PSQL_CMD_SEND;
+}
+
/*
* \t -- turn off table headers and row count
*/
diff --git a/src/bin/psql/common.c b/src/bin/psql/common.c
index f1a5291c13b..b7fe555d56c 100644
--- a/src/bin/psql/common.c
+++ b/src/bin/psql/common.c
@@ -121,6 +121,18 @@ CloseGOutput(FILE *gfile_fout, bool is_pipe)
}
}
+/*
+ * Reset pset pipeline state
+ */
+static void
+pipelineReset(void)
+{
+ pset.piped_syncs = 0;
+ pset.piped_commands = 0;
+ pset.available_results = 0;
+ pset.requested_results = 0;
+}
+
/*
* setQFout
* -- handler for -o command line option and \o command
@@ -354,6 +366,7 @@ CheckConnection(void)
fprintf(stderr, _("The connection to the server was lost. Attempting reset: "));
PQreset(pset.db);
+ pipelineReset();
OK = ConnectionUp();
if (!OK)
{
@@ -415,10 +428,12 @@ AcceptResult(const PGresult *result, bool show_error)
case PGRES_EMPTY_QUERY:
case PGRES_COPY_IN:
case PGRES_COPY_OUT:
+ case PGRES_PIPELINE_SYNC:
/* Fine, do nothing */
OK = true;
break;
+ case PGRES_PIPELINE_ABORTED:
case PGRES_BAD_RESPONSE:
case PGRES_NONFATAL_ERROR:
case PGRES_FATAL_ERROR:
@@ -1050,6 +1065,7 @@ PrintQueryResult(PGresult *result, bool last,
success = true;
break;
+ case PGRES_PIPELINE_ABORTED:
case PGRES_BAD_RESPONSE:
case PGRES_NONFATAL_ERROR:
case PGRES_FATAL_ERROR:
@@ -1418,6 +1434,63 @@ DescribeQuery(const char *query, double *elapsed_msec)
return OK;
}
+/*
+ * Read and discard all results in an aborted pipeline.
+ *
+ * If a synchronisation point is found, we can stop discarding results as
+ * pipeline will switch back to an OK state. If no synchronisation point
+ * is available, we need to stop when there's no more pending results,
+ * otherwise, calling PQgetResult will block.
+ */
+static PGresult *
+discardAbortedPipelineResults(void)
+{
+ for (;;)
+ {
+ PGresult *res = PQgetResult(pset.db);
+ ExecStatusType result_status = PQresultStatus(res);
+
+ if (result_status == PGRES_PIPELINE_SYNC)
+ {
+ /*
+ * Found a synchronisation point. Decrementing the sync counter
+ * will be done by the caller
+ */
+ return res;
+ }
+ else if (res == NULL)
+ {
+ /* A query was processed, decrement the counters */
+ Assert(pset.available_results > 0);
+ Assert(pset.requested_results > 0);
+ pset.available_results--;
+ pset.requested_results--;
+ }
+
+ if (pset.requested_results == 0)
+ {
+ /* We've read all requested results, exit */
+ return res;
+ }
+
+ if (pset.available_results == 0 && pset.piped_syncs == 0)
+ {
+ /*
+ * There's no more results to get and there's no synchronisation
+ * point to stop at. This will leave the pipeline in an aborted
+ * state.
+ */
+ return res;
+ }
+
+ /*
+ * An aborted pipeline will have either NULL results or results in an
+ * PGRES_PIPELINE_ABORTED status
+ */
+ Assert(res == NULL || result_status == PGRES_PIPELINE_ABORTED);
+ PQclear(res);
+ }
+}
/*
* ExecQueryAndProcessResults: utility function for use by SendQuery()
@@ -1451,6 +1524,7 @@ ExecQueryAndProcessResults(const char *query,
bool timing = pset.timing;
bool success = false;
bool return_early = false;
+ bool end_pipeline = false;
instr_time before,
after;
PGresult *result;
@@ -1466,9 +1540,13 @@ ExecQueryAndProcessResults(const char *query,
{
case PSQL_SEND_EXTENDED_CLOSE:
success = PQsendClosePrepared(pset.db, pset.stmtName);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ pset.piped_commands++;
break;
case PSQL_SEND_EXTENDED_PARSE:
success = PQsendPrepare(pset.db, pset.stmtName, query, 0, NULL);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ pset.piped_commands++;
break;
case PSQL_SEND_EXTENDED_QUERY_PARAMS:
Assert(pset.stmtName == NULL);
@@ -1476,6 +1554,8 @@ ExecQueryAndProcessResults(const char *query,
pset.bind_nparams, NULL,
(const char *const *) pset.bind_params,
NULL, NULL, 0);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ pset.piped_commands++;
break;
case PSQL_SEND_EXTENDED_QUERY_PREPARED:
Assert(pset.stmtName != NULL);
@@ -1483,6 +1563,83 @@ ExecQueryAndProcessResults(const char *query,
pset.bind_nparams,
(const char *const *) pset.bind_params,
NULL, NULL, 0);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ pset.piped_commands++;
+ break;
+ case PSQL_SEND_START_PIPELINE_MODE:
+ success = PQenterPipelineMode(pset.db);
+ break;
+ case PSQL_SEND_END_PIPELINE_MODE:
+ success = PQpipelineSync(pset.db);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ /*
+ * End of the pipeline, all queued commands need to be
+ * processed
+ */
+ end_pipeline = true;
+ pset.piped_syncs++;
+
+ /*
+ * The server will send a ReadyForQuery after a Sync is
+ * processed, flushing all results to the client
+ */
+ pset.available_results += pset.piped_commands;
+ pset.piped_commands = 0;
+ /* We want to read all results */
+ pset.requested_results = pset.available_results + pset.piped_syncs;
+ }
+ break;
+ case PSQL_SEND_PIPELINE_SYNC:
+ success = PQsendPipelineSync(pset.db);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pset.piped_syncs++;
+
+ /*
+ * The server will send a ReadyForQuery after a Sync is
+ * processed, flushing all results to the client
+ */
+ pset.available_results += pset.piped_commands;
+ pset.piped_commands = 0;
+ }
+ break;
+ case PSQL_SEND_FLUSH:
+ success = PQflush(pset.db);
+ break;
+ case PSQL_SEND_FLUSH_REQUEST:
+ success = PQsendFlushRequest(pset.db);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ /*
+ * With the flush request, all piped commands are pushed and
+ * the server will forcefully flush the results to the client,
+ * making them available
+ */
+ pset.available_results += pset.piped_commands;
+ pset.piped_commands = 0;
+ }
+ break;
+ case PSQL_SEND_GET_RESULTS:
+ if (pset.available_results == 0 && pset.piped_syncs == 0)
+ {
+ /*
+ * If no sync or flush request were sent, PQgetResult will
+ * block. Forbid the call to \getresults to avoid staying
+ * stuck
+ */
+ pg_log_info("No pending results to get");
+ success = false;
+ pset.requested_results = 0;
+ }
+ else
+ {
+ success = true;
+ /* Cap requested_results to the maximum known results */
+ if (pset.requested_results == 0 ||
+ pset.requested_results > (pset.available_results + pset.piped_syncs))
+ pset.requested_results = pset.available_results + pset.piped_syncs;
+ }
break;
case PSQL_SEND_QUERY:
success = PQsendQuery(pset.db, query);
@@ -1501,6 +1658,16 @@ ExecQueryAndProcessResults(const char *query,
return -1;
}
+ if (pset.requested_results == 0 && !end_pipeline &&
+ PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ /*
+ * We're in a pipeline and haven't reached the pipeline end or there
+ * was no request to read pipeline results, exit.
+ */
+ return 1;
+ }
+
/*
* Fetch the result in chunks if FETCH_COUNT is set, except when:
*
@@ -1548,7 +1715,7 @@ ExecQueryAndProcessResults(const char *query,
{
ExecStatusType result_status;
bool is_chunked_result = false;
- PGresult *next_result;
+ PGresult *next_result = NULL;
bool last;
if (!AcceptResult(result, false))
@@ -1571,6 +1738,9 @@ ExecQueryAndProcessResults(const char *query,
ClearOrSaveResult(result);
success = false;
+ if (result_status == PGRES_PIPELINE_ABORTED)
+ pg_log_info("Pipeline aborted, command didn't run");
+
/*
* switch to next result
*/
@@ -1585,6 +1755,22 @@ ExecQueryAndProcessResults(const char *query,
* ignore manually.
*/
result = NULL;
+ else if ((end_pipeline || pset.requested_results > 0)
+ && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+
+ /*
+ * We have an error within a pipeline. All commands are
+ * aborted until the next synchronisation point. We need to
+ * consume all results until this synchronisation point, or
+ * stop when there's no more result to discard
+ *
+ * Checking pipeline status is necessary in case the
+ * connection was reset. The new connection isn't in any kind
+ * of pipeline state and thus has no result to discard
+ */
+ result = discardAbortedPipelineResults();
+ }
else
result = PQgetResult(pset.db);
@@ -1771,12 +1957,66 @@ ExecQueryAndProcessResults(const char *query,
}
}
+ if (result_status == PGRES_PIPELINE_SYNC)
+ {
+ Assert(pset.piped_syncs > 0);
+
+ /*
+ * We have a sync response, decrease the sync and
+ * requested_results counters
+ */
+ pset.piped_syncs--;
+ pset.requested_results--;
+
+ /*
+ * After a synchronisation point, reset success state to print
+ * possible successful results that will be processed after this
+ */
+ success = true;
+
+ /*
+ * If all syncs were processed and pipeline end was requested,
+ * exit pipeline mode
+ */
+ if (end_pipeline && pset.piped_syncs == 0)
+ success &= PQexitPipelineMode(pset.db);
+ }
+ else if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF &&
+ result_status != PGRES_PIPELINE_SYNC)
+ {
+ /*
+ * We are in a pipeline and have a non sync response, decrease the
+ * results counters
+ */
+ pset.available_results--;
+ pset.requested_results--;
+ }
+
/*
* Check PQgetResult() again. In the typical case of a single-command
* string, it will return NULL. Otherwise, we'll have other results
* to process. We need to do that to check whether this is the last.
*/
- next_result = PQgetResult(pset.db);
+ if (PQpipelineStatus(pset.db) == PQ_PIPELINE_OFF)
+ next_result = PQgetResult(pset.db);
+ else
+ {
+ /*
+ * In pipeline mode, a NULL result indicates the end of the
+ * current query being processed. Call PQgetResult to consume this
+ * NULL
+ */
+ if (result_status != PGRES_PIPELINE_SYNC)
+ {
+ next_result = PQgetResult(pset.db);
+ Assert(next_result == NULL);
+ }
+
+ /* We can now get the next result in the pipeline */
+ if (pset.requested_results > 0)
+ next_result = PQgetResult(pset.db);
+ }
+
last = (next_result == NULL);
/*
@@ -1798,8 +2038,12 @@ ExecQueryAndProcessResults(const char *query,
*elapsed_msec = INSTR_TIME_GET_MILLISEC(after);
}
- /* this may or may not print something depending on settings */
- if (result != NULL)
+ /*
+ * This may or may not print something depending on settings. A
+ * pipeline sync will have a non null result but doesn't have anything
+ * to print, thus ignore them
+ */
+ if (result != NULL && result_status != PGRES_PIPELINE_SYNC)
{
/*
* If results need to be printed into the file specified by \g,
@@ -1825,9 +2069,15 @@ ExecQueryAndProcessResults(const char *query,
ClearOrSaveResult(result);
result = next_result;
- if (cancel_pressed)
+ if (cancel_pressed && PQpipelineStatus(pset.db) == PQ_PIPELINE_OFF)
{
- /* drop this next result, as well as any others not yet read */
+ /*
+ * Outside of a pipeline, drop this next result, as well as any
+ * others not yet read
+ *
+ * Within a pipeline, we can let the outer loop handle this as an
+ * aborted pipeline, which will discard all results
+ */
ClearOrSaveResult(result);
ClearOrSaveAllResults();
break;
@@ -1837,6 +2087,17 @@ ExecQueryAndProcessResults(const char *query,
/* close \g file if we opened it */
CloseGOutput(gfile_fout, gfile_is_pipe);
+ if (end_pipeline)
+ {
+ /* After a pipeline is processed, pipeline piped_syncs should be 0 */
+ Assert(pset.piped_syncs == 0);
+ /* all commands were processed */
+ Assert(pset.piped_commands == 0);
+ /* and all results were read */
+ Assert(pset.available_results == 0);
+ }
+ Assert(pset.requested_results == 0);
+
/* may need this to recover from conn loss during COPY */
if (!CheckConnection())
return -1;
@@ -2297,6 +2558,12 @@ clean_extended_state(void)
pset.bind_params = NULL;
break;
case PSQL_SEND_QUERY:
+ case PSQL_SEND_START_PIPELINE_MODE: /* \startpipeline */
+ case PSQL_SEND_END_PIPELINE_MODE: /* \endpipeline */
+ case PSQL_SEND_PIPELINE_SYNC: /* \syncpipeline */
+ case PSQL_SEND_FLUSH: /* \flush */
+ case PSQL_SEND_GET_RESULTS: /* \getresults */
+ case PSQL_SEND_FLUSH_REQUEST: /* \flushrequest */
break;
}
diff --git a/src/bin/psql/help.c b/src/bin/psql/help.c
index da8e1ade5df..714b8619233 100644
--- a/src/bin/psql/help.c
+++ b/src/bin/psql/help.c
@@ -167,15 +167,22 @@ slashUsage(unsigned short int pager)
HELP0(" \\close STMT_NAME close an existing prepared statement\n");
HELP0(" \\copyright show PostgreSQL usage and distribution terms\n");
HELP0(" \\crosstabview [COLUMNS] execute query and display result in crosstab\n");
+ HELP0(" \\endpipeline exit pipeline mode\n");
HELP0(" \\errverbose show most recent error message at maximum verbosity\n");
+ HELP0(" \\flush push unsent data to the server\n");
+ HELP0(" \\flushrequest send a flushrequest command\n");
HELP0(" \\g [(OPTIONS)] [FILE] execute query (and send result to file or |pipe);\n"
" \\g with no arguments is equivalent to a semicolon\n");
HELP0(" \\gdesc describe result of query, without executing it\n");
+ HELP0(" \\getresults [NUM_RES] read NUM_RES pending results. All pending results are\n"
+ " read if no argument is provided\n");
HELP0(" \\gexec execute query, then execute each value in its result\n");
HELP0(" \\gset [PREFIX] execute query and store result in psql variables\n");
HELP0(" \\gx [(OPTIONS)] [FILE] as \\g, but forces expanded output mode\n");
HELP0(" \\parse STMT_NAME create a prepared statement\n");
HELP0(" \\q quit psql\n");
+ HELP0(" \\startpipeline enter pipeline mode\n");
+ HELP0(" \\syncpipeline add a synchronisation point to an ongoing pipeline\n");
HELP0(" \\watch [[i=]SEC] [c=N] [m=MIN]\n"
" execute query every SEC seconds, up to N times,\n"
" stop if less than MIN rows are returned\n");
diff --git a/src/bin/psql/settings.h b/src/bin/psql/settings.h
index 2a8fe12eb55..022a6e9183e 100644
--- a/src/bin/psql/settings.h
+++ b/src/bin/psql/settings.h
@@ -69,6 +69,12 @@ typedef enum
PSQL_SEND_EXTENDED_PARSE,
PSQL_SEND_EXTENDED_QUERY_PARAMS,
PSQL_SEND_EXTENDED_QUERY_PREPARED,
+ PSQL_SEND_PIPELINE_SYNC,
+ PSQL_SEND_START_PIPELINE_MODE,
+ PSQL_SEND_END_PIPELINE_MODE,
+ PSQL_SEND_FLUSH,
+ PSQL_SEND_FLUSH_REQUEST,
+ PSQL_SEND_GET_RESULTS,
} PSQL_SEND_MODE;
typedef enum
@@ -111,6 +117,12 @@ typedef struct _psqlSettings
char **bind_params; /* parameters for extended query protocol call */
char *stmtName; /* prepared statement name used for extended
* query protocol commands */
+ int piped_commands; /* number of piped commands */
+ int piped_syncs; /* number of piped syncs */
+ int available_results; /* number of results available to get */
+ int requested_results; /* number of requested results, include
+ * sync messages. Used to read a limited
+ * subset of the available_results */
bool crosstab_flag; /* one-shot request to crosstab result */
char *ctv_args[4]; /* \crosstabview arguments */
diff --git a/src/bin/psql/tab-complete.in.c b/src/bin/psql/tab-complete.in.c
index eb8bc128720..8432be641ac 100644
--- a/src/bin/psql/tab-complete.in.c
+++ b/src/bin/psql/tab-complete.in.c
@@ -1885,9 +1885,9 @@ psql_completion(const char *text, int start, int end)
"\\drds", "\\drg", "\\dRs", "\\dRp", "\\ds",
"\\dt", "\\dT", "\\dv", "\\du", "\\dx", "\\dX", "\\dy",
"\\echo", "\\edit", "\\ef", "\\elif", "\\else", "\\encoding",
- "\\endif", "\\errverbose", "\\ev",
- "\\f",
- "\\g", "\\gdesc", "\\getenv", "\\gexec", "\\gset", "\\gx",
+ "\\endif", "\\endpipeline", "\\errverbose", "\\ev",
+ "\\f", "\\flush", "\\flushrequest",
+ "\\g", "\\gdesc", "\\getenv", "\\getresults", "\\gexec", "\\gset", "\\gx",
"\\help", "\\html",
"\\if", "\\include", "\\include_relative", "\\ir",
"\\list", "\\lo_import", "\\lo_export", "\\lo_list", "\\lo_unlink",
@@ -1895,7 +1895,7 @@ psql_completion(const char *text, int start, int end)
"\\parse", "\\password", "\\print", "\\prompt", "\\pset",
"\\qecho", "\\quit",
"\\reset",
- "\\s", "\\set", "\\setenv", "\\sf", "\\sv",
+ "\\s", "\\set", "\\setenv", "\\sf", "\\startpipeline", "\\sv", "\\syncpipeline",
"\\t", "\\T", "\\timing",
"\\unset",
"\\x",
diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule
index e63ee2cf2bb..37b6d21e1f9 100644
--- a/src/test/regress/parallel_schedule
+++ b/src/test/regress/parallel_schedule
@@ -81,7 +81,7 @@ test: brin_bloom brin_multi
test: create_table_like alter_generic alter_operator misc async dbsize merge misc_functions sysviews tsrf tid tidscan tidrangescan collate.utf8 collate.icu.utf8 incremental_sort create_role without_overlaps generated_virtual
# collate.linux.utf8 and collate.icu.utf8 tests cannot be run in parallel with each other
-test: rules psql psql_crosstab amutils stats_ext collate.linux.utf8 collate.windows.win1252
+test: rules psql psql_crosstab psql_pipeline amutils stats_ext collate.linux.utf8 collate.windows.win1252
# ----------
# Run these alone so they don't run out of parallel workers
diff --git a/src/test/regress/sql/psql_pipeline.sql b/src/test/regress/sql/psql_pipeline.sql
new file mode 100644
index 00000000000..84a01d1b72c
--- /dev/null
+++ b/src/test/regress/sql/psql_pipeline.sql
@@ -0,0 +1,401 @@
+--
+-- Tests using psql pipelining
+--
+
+CREATE TABLE psql_pipeline(a INTEGER PRIMARY KEY, s TEXT);
+
+-- single query
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\endpipeline
+
+-- multiple queries
+\startpipeline
+SELECT $1 \bind 'val1' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\endpipeline
+
+-- Test \flush
+\startpipeline
+\flush
+SELECT $1 \bind 'val1' \g
+\flush
+SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\endpipeline
+
+-- send multiple syncs
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\syncpipeline
+\syncpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\syncpipeline
+SELECT $1, $2 \bind 'val4' 'val5' \g
+\endpipeline
+
+-- startpipeline shouldn't have any effect if already in a pipeline
+\startpipeline
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\endpipeline
+
+-- Convert an implicit tx block to an explicit tx block
+\startpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 2 \g
+ROLLBACK \bind \g
+\endpipeline
+
+-- Multiple explicit transactions
+\startpipeline
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+ROLLBACK \bind \g
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+COMMIT \bind \g
+\endpipeline
+
+-- COPY FROM STDIN
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\endpipeline
+2 test2
+\.
+
+-- COPY FROM STDIN with \flushrequest + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\flushrequest
+\getresults
+3 test3
+\.
+\endpipeline
+
+-- COPY FROM STDIN with \syncpipeline + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\syncpipeline
+\getresults
+4 test4
+\.
+\endpipeline
+
+-- COPY TO STDOUT
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\endpipeline
+
+-- COPY TO STDOUT with \flushrequest + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\flushrequest
+\getresults
+\endpipeline
+
+-- COPY TO STDOUT with \syncpipeline + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\syncpipeline
+\getresults
+\endpipeline
+
+-- Use \parse and \bind_named
+\startpipeline
+SELECT $1 \parse ''
+SELECT $1, $2 \parse ''
+SELECT $2 \parse pipeline_1
+\bind_named '' 1 2 \g
+\bind_named pipeline_1 2 \g
+\endpipeline
+
+-- \getresults displays all results preceding a \flushrequest
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+\flushrequest
+\getresults
+\endpipeline
+
+-- \getresults displays all results preceding a sync
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+\syncpipeline
+\getresults
+\endpipeline
+
+-- \getresults immediately returns if there's no result to fetch
+\startpipeline
+\getresults
+SELECT $1 \bind 2 \g
+\getresults
+\flushrequest
+\endpipeline
+\getresults
+
+-- \getresults only fetch results preceding a flushrequest
+\startpipeline
+SELECT $1 \bind 2 \g
+\flushrequest
+SELECT $1 \bind 2 \g
+\getresults
+\endpipeline
+
+-- \getresults only fetch results preceding a sync message
+\startpipeline
+SELECT $1 \bind 2 \g
+\syncpipeline
+SELECT $1 \bind 2 \g
+\getresults
+\endpipeline
+
+-- use pipeline with chunked results with both \getresults and \endpipeline
+\startpipeline
+\set FETCH_COUNT 10
+SELECT $1 \bind 2 \g
+\flushrequest
+\getresults
+SELECT $1 \bind 2 \g
+\endpipeline
+\unset FETCH_COUNT
+
+-- \getresults with specific number of requested results
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+SELECT $1 \bind 3 \g
+\syncpipeline
+\getresults 1
+SELECT $1 \bind 4 \g
+\getresults 3
+\endpipeline
+
+-- \syncpipeline count as one command to fetch for \getresults
+\startpipeline
+\syncpipeline
+\syncpipeline
+SELECT $1 \bind 1 \g
+\flushrequest
+\getresults 2
+\getresults 1
+\endpipeline
+
+-- \getresults 0 should get all results
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+SELECT $1 \bind 3 \g
+\syncpipeline
+\getresults 0
+\endpipeline
+
+-- pipelining errors
+
+-- endpipeline outside of pipeline should fail
+\endpipeline
+
+-- Query using simple protocol should not be sent and should leave the pipeline usable
+\startpipeline
+SELECT 1;
+SELECT $1 \bind 'val1' \g
+\endpipeline
+
+-- After an aborted pipeline, commands after a sync should be displayed
+\startpipeline
+SELECT $1 \bind \g
+\syncpipeline
+SELECT $1 \bind 1 \g
+\endpipeline
+
+-- Incorrect number of parameters, the pipeline will be aborted and following queries won't be executed
+\startpipeline
+SELECT \bind 'val1' \g
+SELECT $1 \bind 'val1' \g
+\endpipeline
+
+-- An explicit transaction with an error needs to be rollbacked after the pipeline
+\startpipeline
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+ROLLBACK \bind \g
+\endpipeline
+ROLLBACK;
+
+-- \watch sends a simple query which won't be allowed within a pipeline
+\startpipeline
+SELECT \bind \g
+\watch 1
+\endpipeline
+
+-- \gdesc should fail as synchronous commands are not allowed in pipeline, pipeline should still be usable
+\startpipeline
+SELECT $1 \bind 1 \gdesc
+SELECT $1 \bind 1 \g
+\endpipeline
+
+-- \gset is not allowed, pipeline should still be usable
+\startpipeline
+SELECT $1 as i, $2 as j \parse ''
+SELECT $1 as k, $2 as l \parse 'second'
+\bind_named '' 1 2 \gset
+\bind_named second 1 2 \gset pref02_ \echo :pref02_i :pref02_j
+\bind_named '' 1 2 \g
+\endpipeline
+
+-- \gx is not allowed, pipeline should still be usable
+\startpipeline
+SELECT $1 \bind 1 \gx
+\reset
+SELECT $1 \bind 1 \g
+\endpipeline
+
+-- \gx warning should be emitted in an aborted pipeline
+\startpipeline
+SELECT $1 \bind \g
+\flushrequest
+\getresults
+SELECT $1 \bind 1 \gx
+\endpipeline
+
+-- \gexec is not allowed, pipeline should still be usable
+\startpipeline
+SELECT 'INSERT INTO psql_pipeline(a) SELECT generate_series(1, 10)' \parse 'insert_stmt'
+\bind_named insert_stmt \gexec
+\bind_named insert_stmt \g
+SELECT COUNT(*) FROM psql_pipeline \bind \g
+\endpipeline
+
+-- After an error, pipeline is aborted and requires a sync to be reusable
+\startpipeline
+SELECT $1 \bind \g
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+\flushrequest
+\getresults
+-- Pipeline is aborted
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+-- Sync allows pipeline to recover
+\syncpipeline
+\getresults
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+\flushrequest
+\getresults
+\endpipeline
+
+-- In an aborted pipeline, \getresults 1 aborted commands one at a time
+\startpipeline
+SELECT $1 \bind \g
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\syncpipeline
+\getresults 1
+\getresults 1
+\getresults 1
+\getresults 1
+\getresults 1
+\endpipeline
+
+-- Test chunked results with an aborted pipeline
+\startpipeline
+\set FETCH_COUNT 10
+SELECT $1 \bind \g
+\flushrequest
+\getresults
+SELECT $1 \bind \g
+\endpipeline
+\unset FETCH_COUNT
+
+-- \getresults returns an error when an incorrect number is provided
+\startpipeline
+\getresults -1
+\endpipeline
+
+-- \getresults when there's no result shouldn't impact the following query
+\getresults 1
+select 1;
+
+-- pipelining and transaction block behaviour
+
+-- set local will issue a warning when modifying a GUC outside of a transaction block
+-- The change will still be valid as a pipeline runs within an implicit transaction block
+-- Sending a sync will commit the implicit transaction block. The first command after a sync
+-- won't be seen as belonging to a pipeline.
+\startpipeline
+SET LOCAL statement_timeout='1h' \bind \g
+SHOW statement_timeout \bind \g
+\syncpipeline
+SHOW statement_timeout \bind \g
+SET LOCAL statement_timeout='2h' \bind \g
+SHOW statement_timeout \bind \g
+\endpipeline
+
+-- Reindex concurrently is forbidden in the middle of a pipeline
+\startpipeline
+SELECT $1 \bind 1 \g
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- Reindex concurrently will work if it's the first command of a pipeline
+\startpipeline
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- subtransactions are not allowed in pipeline mode
+\startpipeline
+SAVEPOINT a \bind \g
+SELECT $1 \bind 1 \g
+ROLLBACK TO SAVEPOINT a \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- Lock command will fail as first pipeline command is not seen as a transaction block
+\startpipeline
+LOCK psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- Lock command will succeed after the first command as pipeline will be seen as an implicit transaction block
+\startpipeline
+SELECT $1 \bind 1 \g
+LOCK psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- Vacuum command will work as the first command
+\startpipeline
+VACUUM psql_pipeline \bind \g
+\endpipeline
+
+-- Vacuum command will fail within pipeline implicit transaction
+\startpipeline
+SELECT 1 \bind \g
+VACUUM psql_pipeline \bind \g
+\endpipeline
+
+-- Vacuum command will work after a sync
+\startpipeline
+SELECT 1 \bind \g
+\syncpipeline
+VACUUM psql_pipeline \bind \g
+\endpipeline
--
2.39.5 (Apple Git-154)
[application/octet-stream] v08-0002-Add-prompt-interpolation-and-variables-for-psql-.patch (6.7K, ../../CAO6_XqrrjJRhKNhYa9Cg=_BSH8MmS1tAHMFq-H7fH=NcWJ-jvw@mail.gmail.com/3-v08-0002-Add-prompt-interpolation-and-variables-for-psql-.patch)
download | inline diff:
From 06962deb46be6af15dfe694e6e6342af02964638 Mon Sep 17 00:00:00 2001
From: Anthonin Bonnefoy <[email protected]>
Date: Tue, 18 Feb 2025 15:32:31 +0100
Subject: Add prompt interpolation and variables for psql pipeline
Add %P prompt interpolation that reports the status pipeline: on, off or
abort. Additionally, 3 new special variables are added to report the
status of an ongoing pipeline:
- PIPELINE_SYNC_COUNT: reports the number of piped sync
- PIPELINE_COMMAND_COUNT: reports the number of piped commands, a
command being either \bind, \bind_named, \close or \parse
- PIPELINE_RESULT_COUNT: reports the results available to read with
\getresults
---
doc/src/sgml/ref/psql-ref.sgml | 49 ++++++++++++++++++++++++++++++++++
src/bin/psql/common.c | 27 ++++++++++++++++++-
src/bin/psql/prompt.c | 12 +++++++++
src/bin/psql/startup.c | 5 ++++
4 files changed, 92 insertions(+), 1 deletion(-)
diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml
index fdae1ed0479..091cbaefc15 100644
--- a/doc/src/sgml/ref/psql-ref.sgml
+++ b/doc/src/sgml/ref/psql-ref.sgml
@@ -3723,6 +3723,12 @@ testdb=> <userinput>\setenv LESS -imx4F</userinput>
generate one result to get.
</para>
+ <para>
+ When pipeline mode is active, a dedicated prompt variable is
+ available to report the pipeline status. See
+ <xref linkend="app-psql-prompting-p-uc"/> for more details
+ </para>
+
<para>
Example:
<programlisting>
@@ -4504,6 +4510,38 @@ bar
</listitem>
</varlistentry>
+ <varlistentry id="app-psql-variables-pipeline-command-count">
+ <term><varname>PIPELINE_COMMAND_COUNT</varname></term>
+ <listitem>
+ <para>
+ The number of commands generated by <literal>\bind</literal>,
+ <literal>\bind_named</literal>, <literal>\close</literal> or
+ <literal>\parse</literal> queued in an ongoing pipeline.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry id="app-psql-variables-pipeline-result-count">
+ <term><varname>PIPELINE_RESULT_COUNT</varname></term>
+ <listitem>
+ <para>
+ The number of commands of an ongoing pipeline that were followed
+ by either a <command>\flushrequest</command> or a
+ <command>\syncpipeline</command>, forcing the server to send the
+ results and can be retrieved with <command>\getresults</command>.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry id="app-psql-variables-pipeline-sync-count">
+ <term><varname>PIPELINE_SYNC_COUNT</varname></term>
+ <listitem>
+ <para>
+ The number of syncs queued in an ongoing pipeline.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry id="app-psql-variables-port">
<term><varname>PORT</varname></term>
<listitem>
@@ -4903,6 +4941,17 @@ testdb=> <userinput>INSERT INTO my_table VALUES (:'content');</userinput>
</listitem>
</varlistentry>
+ <varlistentry id="app-psql-prompting-p-uc">
+ <term><literal>%P</literal></term>
+ <listitem>
+ <para>
+ Pipeline status: <literal>off</literal> when not in a pipeline,
+ or <literal>on</literal> when in an ongoing pipeline, or
+ <literal>abort</literal> when in an aborted pipeline.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry id="app-psql-prompting-r">
<term><literal>%R</literal></term>
<listitem>
diff --git a/src/bin/psql/common.c b/src/bin/psql/common.c
index b7fe555d56c..70200e8594d 100644
--- a/src/bin/psql/common.c
+++ b/src/bin/psql/common.c
@@ -524,6 +524,27 @@ SetShellResultVariables(int wait_result)
}
+/*
+ * Set special pipeline variables
+ * - PIPELINE_SYNC_COUNT: The number of piped syncs
+ * - PIPELINE_COMMAND_COUNT: The number of piped commands
+ * - PIPELINE_RESULT_COUNT: The number of results available to read
+ */
+static void
+SetPipelineVariables(void)
+{
+ char buf[32];
+
+ snprintf(buf, sizeof(buf), "%d", pset.piped_syncs);
+ SetVariable(pset.vars, "PIPELINE_SYNC_COUNT", buf);
+ snprintf(buf, sizeof(buf), "%d", pset.piped_commands);
+ SetVariable(pset.vars, "PIPELINE_COMMAND_COUNT", buf);
+ snprintf(buf, sizeof(buf), "%d", pset.available_results);
+ SetVariable(pset.vars, "PIPELINE_RESULT_COUNT", buf);
+ Assert(pset.requested_results == 0);
+}
+
+
/*
* ClearOrSaveResult
*
@@ -1655,6 +1676,8 @@ ExecQueryAndProcessResults(const char *query,
CheckConnection();
+ SetPipelineVariables();
+
return -1;
}
@@ -1663,8 +1686,9 @@ ExecQueryAndProcessResults(const char *query,
{
/*
* We're in a pipeline and haven't reached the pipeline end or there
- * was no request to read pipeline results, exit.
+ * was no request to read pipeline results, update psql variables and exit.
*/
+ SetPipelineVariables();
return 1;
}
@@ -2097,6 +2121,7 @@ ExecQueryAndProcessResults(const char *query,
Assert(pset.available_results == 0);
}
Assert(pset.requested_results == 0);
+ SetPipelineVariables();
/* may need this to recover from conn loss during COPY */
if (!CheckConnection())
diff --git a/src/bin/psql/prompt.c b/src/bin/psql/prompt.c
index 08a14feb3c3..78505222e01 100644
--- a/src/bin/psql/prompt.c
+++ b/src/bin/psql/prompt.c
@@ -31,6 +31,7 @@
* sockets, "[local:/dir/name]" if not default
* %m - like %M, but hostname only (before first dot), or always "[local]"
* %p - backend pid
+ * %P - pipeline status: on, off or abort
* %> - database server port number
* %n - database user name
* %s - service
@@ -181,7 +182,18 @@ get_prompt(promptStatus_t status, ConditionalStack cstack)
snprintf(buf, sizeof(buf), "%d", pid);
}
break;
+ case 'P':
+ {
+ PGpipelineStatus status = PQpipelineStatus(pset.db);
+ if (status == PQ_PIPELINE_ON)
+ strlcpy(buf, "on", sizeof(buf));
+ else if (status == PQ_PIPELINE_ABORTED)
+ strlcpy(buf, "abort", sizeof(buf));
+ else
+ strlcpy(buf, "off", sizeof(buf));
+ break;
+ }
case '0':
case '1':
case '2':
diff --git a/src/bin/psql/startup.c b/src/bin/psql/startup.c
index 703f3f582c1..5018eedf1e5 100644
--- a/src/bin/psql/startup.c
+++ b/src/bin/psql/startup.c
@@ -205,6 +205,11 @@ main(int argc, char *argv[])
SetVariable(pset.vars, "PROMPT3", DEFAULT_PROMPT3);
SetVariableBool(pset.vars, "SHOW_ALL_RESULTS");
+ /* Initialize pipeline variables */
+ SetVariable(pset.vars, "PIPELINE_SYNC_COUNT", "0");
+ SetVariable(pset.vars, "PIPELINE_COMMAND_COUNT", "0");
+ SetVariable(pset.vars, "PIPELINE_RESULT_COUNT", "0");
+
parse_psql_options(argc, argv, &options);
/*
--
2.39.5 (Apple Git-154)
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-02-20 08:02 Michael Paquier <[email protected]>
parent: Anthonin Bonnefoy <[email protected]>
1 sibling, 1 reply; 41+ messages in thread
From: Michael Paquier @ 2025-02-20 08:02 UTC (permalink / raw)
To: Anthonin Bonnefoy <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; pgsql-hackers
On Tue, Feb 18, 2025 at 06:34:20PM +0100, Anthonin Bonnefoy wrote:
> On Tue, Feb 18, 2025 at 8:23 AM Michael Paquier <[email protected]> wrote:
>> The tests in psql.sql are becoming really long. Perhaps it would be
>> better to split that into its own file, say psql_pipeline.sql? The
>> input file is already 2k lines, you are adding 15% more lines to that.
>
> Agreed, I wasn't sure if this was enough to warrant a dedicated test
> file. This is now separated in psql_pipeline.sql.
You have forgotten the expected output. Not a big issue as the input
was sent.
>> What is the reasoning here behind this restriction? \gx is a wrapper
>> of \g with expanded mode on, but it is also possible to call \g with
>> expanded=on, bypassing this restriction.
>
> The issue is that \gx enables expanded mode for the duration of the
> query and immediately reset it in sendquery_cleanup. With pipelining,
> the command is piped and displaying is done by either \endpipeline or
> \getresults, so the flag change has no impact. Forbidding it was a way
> to make it clearer that it won't have the expected effect. If we
> wanted a similar feature, this would need to be done with something
> like \endpipelinex or \getresultsx.
Hmm, okay. If one wants one mode or the other it is always possible
to force one with \pset expanded when getting the results. Not sure
if there is any need for new specific commands for these two printing
the results. Another option would be to authorize the command to run,
but perhaps your option is just better as per the enforced behavior in
the output. So fine by me. There is coverage so we'll know if there
are arguments in favor of authorizing the command, if need be.
> I've split the patch and created the 3 special variables:
> PIPELINE_SYNC_COUNT, PIPELINE_COMMAND_COUNT, PIPELINE_RESULT_COUNT.
Thanks. Looks sensible now.
> For requested_results, I don't think there's value in exposing it
> since it is used as an exit condition and thus will always be 0
> outside of ExecQueryAndProcessResults.
I've been playing with this patch and this configuration:
\set PROMPT1 '=(pipeline=%P,sync=%:PIPELINE_SYNC_COUNT:,cmd=%:PIPELINE_COMMAND_COUNT:,res=%:PIPELINE_RESULT_COUNT:)%#'
That's long, but seeing the evolution of the pipeline status is pretty
cool depending on the meta-commands used.
While testing, I have been able to run into an assertion failure by
adding some tests in psql.sql to check for the case of inactive
branches for \if. For example:
--- a/src/test/regress/sql/psql.sql
+++ b/src/test/regress/sql/psql.sql
@@ -1047,11 +1047,15 @@ select \if false \\ (bogus \else \\ 42 \endif \\ forty_two;
\echo arg1 arg2 arg3 arg4 arg5
\echo arg1
\encoding arg1
+ \endpipeline
\errverbose
And the report:
+psql: mainloop.c:513: MainLoop: Assertion `conditional_active(cond_stack)' failed.
We should have tests for all new six meta-commands in psql.sql.
MainLoop() is wrong when in pipeline mode for inactive branches.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-02-20 09:29 Anthonin Bonnefoy <[email protected]>
parent: Michael Paquier <[email protected]>
0 siblings, 1 reply; 41+ messages in thread
From: Anthonin Bonnefoy @ 2025-02-20 09:29 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; pgsql-hackers
On Thu, Feb 20, 2025 at 9:02 AM Michael Paquier <[email protected]> wrote:
> You have forgotten the expected output. Not a big issue as the input
> was sent.
I was writing the mail with the missing file when you sent this mail.
This is fixed.
> While testing, I have been able to run into an assertion failure by
> adding some tests in psql.sql to check for the case of inactive
> branches for \if. For example:
> --- a/src/test/regress/sql/psql.sql
> +++ b/src/test/regress/sql/psql.sql
> @@ -1047,11 +1047,15 @@ select \if false \\ (bogus \else \\ 42 \endif \\ forty_two;
> \echo arg1 arg2 arg3 arg4 arg5
> \echo arg1
> \encoding arg1
> + \endpipeline
> \errverbose
>
> And the report:
> +psql: mainloop.c:513: MainLoop: Assertion `conditional_active(cond_stack)' failed.
>
> We should have tests for all new six meta-commands in psql.sql.
> MainLoop() is wrong when in pipeline mode for inactive branches.
Ha yeah, I forgot about the inactive branches. I've added the new
commands and fixed the behaviour.
A small issue I've noticed while testing: When a pipeline has at least
one queue command, pqClearConnErrorState isn't called in
PQsendQueryStart and errors are appended. For example:
\startpipeline
select 1 \bind \g
select 1;
PQsendQuery not allowed in pipeline mode
select 1;
PQsendQuery not allowed in pipeline mode
PQsendQuery not allowed in pipeline mode
This looks more like an issue on libpq's side as there's no way to
reset or advance the errorReported from ExecQueryAndProcessResults
(plus PQerrorMessage seems to ignore errorReported). I've added an
additional test to track this behaviour for now as this would probably
be better discussed in a dedicated thread.
Attachments:
[application/octet-stream] v09-0002-Add-prompt-interpolation-and-variables-for-psql-.patch (6.7K, ../../CAO6_Xqr86Pc2qyk0pydfqkmrb63k9rw49T0dsS=WFB8x5oMM4Q@mail.gmail.com/2-v09-0002-Add-prompt-interpolation-and-variables-for-psql-.patch)
download | inline diff:
From 74cfea223bd46af9af610d615076151a8d6f8761 Mon Sep 17 00:00:00 2001
From: Anthonin Bonnefoy <[email protected]>
Date: Tue, 18 Feb 2025 15:32:31 +0100
Subject: Add prompt interpolation and variables for psql pipeline
Add %P prompt interpolation that reports the status pipeline: on, off or
abort. Additionally, 3 new special variables are added to report the
status of an ongoing pipeline:
- PIPELINE_SYNC_COUNT: reports the number of piped sync
- PIPELINE_COMMAND_COUNT: reports the number of piped commands, a
command being either \bind, \bind_named, \close or \parse
- PIPELINE_RESULT_COUNT: reports the results available to read with
\getresults
---
doc/src/sgml/ref/psql-ref.sgml | 49 ++++++++++++++++++++++++++++++++++
src/bin/psql/common.c | 27 ++++++++++++++++++-
src/bin/psql/prompt.c | 12 +++++++++
src/bin/psql/startup.c | 5 ++++
4 files changed, 92 insertions(+), 1 deletion(-)
diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml
index fdae1ed0479..091cbaefc15 100644
--- a/doc/src/sgml/ref/psql-ref.sgml
+++ b/doc/src/sgml/ref/psql-ref.sgml
@@ -3723,6 +3723,12 @@ testdb=> <userinput>\setenv LESS -imx4F</userinput>
generate one result to get.
</para>
+ <para>
+ When pipeline mode is active, a dedicated prompt variable is
+ available to report the pipeline status. See
+ <xref linkend="app-psql-prompting-p-uc"/> for more details
+ </para>
+
<para>
Example:
<programlisting>
@@ -4504,6 +4510,38 @@ bar
</listitem>
</varlistentry>
+ <varlistentry id="app-psql-variables-pipeline-command-count">
+ <term><varname>PIPELINE_COMMAND_COUNT</varname></term>
+ <listitem>
+ <para>
+ The number of commands generated by <literal>\bind</literal>,
+ <literal>\bind_named</literal>, <literal>\close</literal> or
+ <literal>\parse</literal> queued in an ongoing pipeline.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry id="app-psql-variables-pipeline-result-count">
+ <term><varname>PIPELINE_RESULT_COUNT</varname></term>
+ <listitem>
+ <para>
+ The number of commands of an ongoing pipeline that were followed
+ by either a <command>\flushrequest</command> or a
+ <command>\syncpipeline</command>, forcing the server to send the
+ results and can be retrieved with <command>\getresults</command>.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry id="app-psql-variables-pipeline-sync-count">
+ <term><varname>PIPELINE_SYNC_COUNT</varname></term>
+ <listitem>
+ <para>
+ The number of syncs queued in an ongoing pipeline.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry id="app-psql-variables-port">
<term><varname>PORT</varname></term>
<listitem>
@@ -4903,6 +4941,17 @@ testdb=> <userinput>INSERT INTO my_table VALUES (:'content');</userinput>
</listitem>
</varlistentry>
+ <varlistentry id="app-psql-prompting-p-uc">
+ <term><literal>%P</literal></term>
+ <listitem>
+ <para>
+ Pipeline status: <literal>off</literal> when not in a pipeline,
+ or <literal>on</literal> when in an ongoing pipeline, or
+ <literal>abort</literal> when in an aborted pipeline.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry id="app-psql-prompting-r">
<term><literal>%R</literal></term>
<listitem>
diff --git a/src/bin/psql/common.c b/src/bin/psql/common.c
index b7fe555d56c..3d67ad793bf 100644
--- a/src/bin/psql/common.c
+++ b/src/bin/psql/common.c
@@ -524,6 +524,26 @@ SetShellResultVariables(int wait_result)
}
+/*
+ * Set special pipeline variables
+ * - PIPELINE_SYNC_COUNT: The number of piped syncs
+ * - PIPELINE_COMMAND_COUNT: The number of piped commands
+ * - PIPELINE_RESULT_COUNT: The number of results available to read
+ */
+static void
+SetPipelineVariables(void)
+{
+ char buf[32];
+
+ snprintf(buf, sizeof(buf), "%d", pset.piped_syncs);
+ SetVariable(pset.vars, "PIPELINE_SYNC_COUNT", buf);
+ snprintf(buf, sizeof(buf), "%d", pset.piped_commands);
+ SetVariable(pset.vars, "PIPELINE_COMMAND_COUNT", buf);
+ snprintf(buf, sizeof(buf), "%d", pset.available_results);
+ SetVariable(pset.vars, "PIPELINE_RESULT_COUNT", buf);
+}
+
+
/*
* ClearOrSaveResult
*
@@ -1655,6 +1675,8 @@ ExecQueryAndProcessResults(const char *query,
CheckConnection();
+ SetPipelineVariables();
+
return -1;
}
@@ -1663,8 +1685,10 @@ ExecQueryAndProcessResults(const char *query,
{
/*
* We're in a pipeline and haven't reached the pipeline end or there
- * was no request to read pipeline results, exit.
+ * was no request to read pipeline results, update psql variables and
+ * exit.
*/
+ SetPipelineVariables();
return 1;
}
@@ -2097,6 +2121,7 @@ ExecQueryAndProcessResults(const char *query,
Assert(pset.available_results == 0);
}
Assert(pset.requested_results == 0);
+ SetPipelineVariables();
/* may need this to recover from conn loss during COPY */
if (!CheckConnection())
diff --git a/src/bin/psql/prompt.c b/src/bin/psql/prompt.c
index 08a14feb3c3..78505222e01 100644
--- a/src/bin/psql/prompt.c
+++ b/src/bin/psql/prompt.c
@@ -31,6 +31,7 @@
* sockets, "[local:/dir/name]" if not default
* %m - like %M, but hostname only (before first dot), or always "[local]"
* %p - backend pid
+ * %P - pipeline status: on, off or abort
* %> - database server port number
* %n - database user name
* %s - service
@@ -181,7 +182,18 @@ get_prompt(promptStatus_t status, ConditionalStack cstack)
snprintf(buf, sizeof(buf), "%d", pid);
}
break;
+ case 'P':
+ {
+ PGpipelineStatus status = PQpipelineStatus(pset.db);
+ if (status == PQ_PIPELINE_ON)
+ strlcpy(buf, "on", sizeof(buf));
+ else if (status == PQ_PIPELINE_ABORTED)
+ strlcpy(buf, "abort", sizeof(buf));
+ else
+ strlcpy(buf, "off", sizeof(buf));
+ break;
+ }
case '0':
case '1':
case '2':
diff --git a/src/bin/psql/startup.c b/src/bin/psql/startup.c
index 703f3f582c1..5018eedf1e5 100644
--- a/src/bin/psql/startup.c
+++ b/src/bin/psql/startup.c
@@ -205,6 +205,11 @@ main(int argc, char *argv[])
SetVariable(pset.vars, "PROMPT3", DEFAULT_PROMPT3);
SetVariableBool(pset.vars, "SHOW_ALL_RESULTS");
+ /* Initialize pipeline variables */
+ SetVariable(pset.vars, "PIPELINE_SYNC_COUNT", "0");
+ SetVariable(pset.vars, "PIPELINE_COMMAND_COUNT", "0");
+ SetVariable(pset.vars, "PIPELINE_RESULT_COUNT", "0");
+
parse_psql_options(argc, argv, &options);
/*
--
2.39.5 (Apple Git-154)
[application/octet-stream] v09-0001-Add-pipelining-support-in-psql.patch (56.2K, ../../CAO6_Xqr86Pc2qyk0pydfqkmrb63k9rw49T0dsS=WFB8x5oMM4Q@mail.gmail.com/3-v09-0001-Add-pipelining-support-in-psql.patch)
download | inline diff:
From b0b28575e2b2da027cfc0a37e92da4b9e077b138 Mon Sep 17 00:00:00 2001
From: Anthonin Bonnefoy <[email protected]>
Date: Tue, 5 Nov 2024 10:26:54 +0100
Subject: Add pipelining support in psql
With \bind, \parse, \bind_named and \close, it is possible to issue
queries from psql using the extended protocol. However, it wasn't
possible to send those queries using pipelining and the only way to test
pipelined queries was through pgbench's tap tests.
This patch adds additional psql meta-commands to support pipelining:
\startpipeline, \endpipeline and \syncpipeline, mirroring the existing
meta-commands in pgbench. Additional meta-commands allow to flush and
read results of an ongoing pipeline: \flushrequest, \flush and \getresults
\startpipeline starts a new pipeline. All extended queries will be
queued until the end of the pipeline is reached.
\endpipeline ends an ongoing pipeline. All queued commands will be sent
to the server and all responses will be processed by the psql.
\syncpipeline queues a synchronisation point without flushing the
commands to the server
\flush Call PQflush on psql's connection
\flushrequest queues a flushrequest
\getresults reads server's results. Unsent data are automatically pushed
when \getresults is called
Those meta-commands will allow to test pipeline behaviour using
psql regression tests.
---
doc/src/sgml/ref/psql-ref.sgml | 70 ++
src/bin/psql/command.c | 169 +++++
src/bin/psql/common.c | 279 +++++++-
src/bin/psql/help.c | 7 +
src/bin/psql/settings.h | 12 +
src/bin/psql/tab-complete.in.c | 8 +-
src/test/regress/expected/psql.out | 6 +
src/test/regress/expected/psql_pipeline.out | 684 ++++++++++++++++++++
src/test/regress/parallel_schedule | 2 +-
src/test/regress/sql/psql.sql | 6 +
src/test/regress/sql/psql_pipeline.sql | 408 ++++++++++++
11 files changed, 1640 insertions(+), 11 deletions(-)
create mode 100644 src/test/regress/expected/psql_pipeline.out
create mode 100644 src/test/regress/sql/psql_pipeline.sql
diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml
index f3044fac1fa..fdae1ed0479 100644
--- a/doc/src/sgml/ref/psql-ref.sgml
+++ b/doc/src/sgml/ref/psql-ref.sgml
@@ -3674,6 +3674,76 @@ testdb=> <userinput>\setenv LESS -imx4F</userinput>
</listitem>
</varlistentry>
+ <varlistentry id="app-psql-meta-command-pipeline">
+ <term><literal>\startpipeline</literal></term>
+ <term><literal>\syncpipeline</literal></term>
+ <term><literal>\endpipeline</literal></term>
+ <term><literal>\flushrequest</literal></term>
+ <term><literal>\flush</literal></term>
+ <term><literal>\getresults [ <replaceable class="parameter">number_results</replaceable> ]</literal></term>
+
+ <listitem>
+ <para>
+ This group of commands implements pipelining of SQL statements.
+ A pipeline must begin with a <command>\startpipeline</command>
+ and end with an <command>\endpipeline</command>. In between there
+ may be any number of <command>\syncpipeline</command> commands,
+ which sends a <link linkend="protocol-flow-ext-query">sync message</link>
+ without ending the ongoing pipeline and flushing the send buffer.
+ In pipeline mode, statements are sent to the server without waiting
+ for the results of previous statements. See
+ <xref linkend="libpq-pipeline-mode"/> for more details.
+ </para>
+
+ <para>
+ Pipeline mode requires the use of extended query protocol. All queries need
+ to be sent using the meta-commands <literal>\bind</literal>,
+ <literal>\bind_named</literal>, <literal>\close</literal> or
+ <literal>\parse</literal>. While a pipeline is ongoing,
+ <literal>\g</literal> will append the current query buffer to the pipeline and
+ other meta-commands like <literal>\gx</literal> or <literal>\gdesc</literal>
+ are not allowed in pipeline mode.
+ </para>
+
+ <para>
+ <command>\flushrequest</command> appends a flush command to the pipeline,
+ allowing to read results with <command>\getresults</command> without issuing
+ a sync or ending the pipeline. <command>\getresults</command> will automatically
+ push unsent data to the server. <command>\flush</command> can be used to manually
+ push unsent data.
+ </para>
+
+ <para>
+ <command>\getresults</command> accepts an optional
+ <replaceable class="parameter">number_results</replaceable> parameter. If provided,
+ only the first <replaceable class="parameter">number_results</replaceable> pending
+ results will be read. If not provided or 0, all pending results are read. The
+ commands <literal>\bind</literal>, <literal>\bind_named</literal>,
+ <literal>\close</literal>, <literal>\parse</literal> and <literal>\syncpipeline</literal>
+ generate one result to get.
+ </para>
+
+ <para>
+ Example:
+<programlisting>
+\startpipeline
+SELECT 1 \bind \g
+SELECT $1 \parse stmt1
+\bind_named stmt1 1 \g
+SELECT pg_current_xact_id() \bind \g
+\flushrequest
+\getresults
+\syncpipeline
+SELECT pg_current_xact_id() \bind \g
+\flushrequest
+\getresults 2
+\close stmt1
+\endpipeline
+</programlisting></para>
+
+ </listitem>
+ </varlistentry>
+
<varlistentry id="app-psql-meta-command-t-lc">
<term><literal>\t</literal></term>
diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c
index 26dfdde195a..1c62a54c06b 100644
--- a/src/bin/psql/command.c
+++ b/src/bin/psql/command.c
@@ -90,9 +90,12 @@ static backslashResult exec_command_else(PsqlScanState scan_state, ConditionalSt
PQExpBuffer query_buf);
static backslashResult exec_command_endif(PsqlScanState scan_state, ConditionalStack cstack,
PQExpBuffer query_buf);
+static backslashResult exec_command_endpipeline(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_encoding(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_errverbose(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_f(PsqlScanState scan_state, bool active_branch);
+static backslashResult exec_command_flush(PsqlScanState scan_state, bool active_branch);
+static backslashResult exec_command_flushrequest(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_g(PsqlScanState scan_state, bool active_branch,
const char *cmd);
static backslashResult process_command_g_options(char *first_option,
@@ -103,6 +106,7 @@ static backslashResult exec_command_gdesc(PsqlScanState scan_state, bool active_
static backslashResult exec_command_getenv(PsqlScanState scan_state, bool active_branch,
const char *cmd);
static backslashResult exec_command_gexec(PsqlScanState scan_state, bool active_branch);
+static backslashResult exec_command_getresults(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_gset(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_help(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_html(PsqlScanState scan_state, bool active_branch);
@@ -132,6 +136,8 @@ static backslashResult exec_command_setenv(PsqlScanState scan_state, bool active
const char *cmd);
static backslashResult exec_command_sf_sv(PsqlScanState scan_state, bool active_branch,
const char *cmd, bool is_func);
+static backslashResult exec_command_startpipeline(PsqlScanState scan_state, bool active_branch);
+static backslashResult exec_command_syncpipeline(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_t(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_T(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_timing(PsqlScanState scan_state, bool active_branch);
@@ -351,18 +357,26 @@ exec_command(const char *cmd,
status = exec_command_else(scan_state, cstack, query_buf);
else if (strcmp(cmd, "endif") == 0)
status = exec_command_endif(scan_state, cstack, query_buf);
+ else if (strcmp(cmd, "endpipeline") == 0)
+ status = exec_command_endpipeline(scan_state, active_branch);
else if (strcmp(cmd, "encoding") == 0)
status = exec_command_encoding(scan_state, active_branch);
else if (strcmp(cmd, "errverbose") == 0)
status = exec_command_errverbose(scan_state, active_branch);
else if (strcmp(cmd, "f") == 0)
status = exec_command_f(scan_state, active_branch);
+ else if (strcmp(cmd, "flush") == 0)
+ status = exec_command_flush(scan_state, active_branch);
+ else if (strcmp(cmd, "flushrequest") == 0)
+ status = exec_command_flushrequest(scan_state, active_branch);
else if (strcmp(cmd, "g") == 0 || strcmp(cmd, "gx") == 0)
status = exec_command_g(scan_state, active_branch, cmd);
else if (strcmp(cmd, "gdesc") == 0)
status = exec_command_gdesc(scan_state, active_branch);
else if (strcmp(cmd, "getenv") == 0)
status = exec_command_getenv(scan_state, active_branch, cmd);
+ else if (strcmp(cmd, "getresults") == 0)
+ status = exec_command_getresults(scan_state, active_branch);
else if (strcmp(cmd, "gexec") == 0)
status = exec_command_gexec(scan_state, active_branch);
else if (strcmp(cmd, "gset") == 0)
@@ -411,6 +425,10 @@ exec_command(const char *cmd,
status = exec_command_sf_sv(scan_state, active_branch, cmd, true);
else if (strcmp(cmd, "sv") == 0 || strcmp(cmd, "sv+") == 0)
status = exec_command_sf_sv(scan_state, active_branch, cmd, false);
+ else if (strcmp(cmd, "startpipeline") == 0)
+ status = exec_command_startpipeline(scan_state, active_branch);
+ else if (strcmp(cmd, "syncpipeline") == 0)
+ status = exec_command_syncpipeline(scan_state, active_branch);
else if (strcmp(cmd, "t") == 0)
status = exec_command_t(scan_state, active_branch);
else if (strcmp(cmd, "T") == 0)
@@ -1515,6 +1533,44 @@ exec_command_f(PsqlScanState scan_state, bool active_branch)
return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR;
}
+/*
+ * \flush -- call PQflush on the connection
+ */
+static backslashResult
+exec_command_flush(PsqlScanState scan_state, bool active_branch)
+{
+ backslashResult status = PSQL_CMD_SKIP_LINE;
+
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_SEND_FLUSH;
+ status = PSQL_CMD_SEND;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return status;
+}
+
+/*
+ * \flushrequest -- send a flush request to the server
+ */
+static backslashResult
+exec_command_flushrequest(PsqlScanState scan_state, bool active_branch)
+{
+ backslashResult status = PSQL_CMD_SKIP_LINE;
+
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_SEND_FLUSH_REQUEST;
+ status = PSQL_CMD_SEND;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return status;
+}
+
/*
* \g [(pset-option[=pset-value] ...)] [filename/shell-command]
* \gx [(pset-option[=pset-value] ...)] [filename/shell-command]
@@ -1550,6 +1606,13 @@ exec_command_g(PsqlScanState scan_state, bool active_branch, const char *cmd)
if (status == PSQL_CMD_SKIP_LINE && active_branch)
{
+ if (strcmp(cmd, "gx") == 0 && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pg_log_error("\\gx not allowed in pipeline mode");
+ clean_extended_state();
+ return PSQL_CMD_ERROR;
+ }
+
if (!fname)
pset.gfname = NULL;
else
@@ -1703,6 +1766,42 @@ exec_command_getenv(PsqlScanState scan_state, bool active_branch,
return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR;
}
+/*
+ * \getresults -- read results
+ */
+static backslashResult
+exec_command_getresults(PsqlScanState scan_state, bool active_branch)
+{
+ backslashResult status = PSQL_CMD_SKIP_LINE;
+
+ if (active_branch)
+ {
+ char *opt;
+ int num_results;
+
+ pset.send_mode = PSQL_SEND_GET_RESULTS;
+ status = PSQL_CMD_SEND;
+ opt = psql_scan_slash_option(scan_state, OT_NORMAL, NULL, false);
+
+ pset.requested_results = 0;
+ if (opt != NULL)
+ {
+ num_results = atoi(opt);
+ if (num_results < 0)
+ {
+ pg_log_error("\\getresults: invalid number of requested results");
+ return PSQL_CMD_SKIP_LINE;
+ }
+ pset.requested_results = num_results;
+ }
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return status;
+}
+
+
/*
* \gexec -- send query and execute each field of result
*/
@@ -1713,6 +1812,12 @@ exec_command_gexec(PsqlScanState scan_state, bool active_branch)
if (active_branch)
{
+ if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pg_log_error("\\gexec not allowed in pipeline mode");
+ clean_extended_state();
+ return PSQL_CMD_ERROR;
+ }
pset.gexec_flag = true;
status = PSQL_CMD_SEND;
}
@@ -1733,6 +1838,13 @@ exec_command_gset(PsqlScanState scan_state, bool active_branch)
char *prefix = psql_scan_slash_option(scan_state,
OT_NORMAL, NULL, false);
+ if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pg_log_error("\\gset not allowed in pipeline mode");
+ clean_extended_state();
+ return PSQL_CMD_ERROR;
+ }
+
if (prefix)
pset.gset_prefix = prefix;
else
@@ -2718,6 +2830,63 @@ exec_command_sf_sv(PsqlScanState scan_state, bool active_branch,
return status;
}
+/*
+ * \startpipeline -- enter pipeline mode
+ */
+static backslashResult
+exec_command_startpipeline(PsqlScanState scan_state, bool active_branch)
+{
+ backslashResult status = PSQL_CMD_SKIP_LINE;
+
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_SEND_START_PIPELINE_MODE;
+ status = PSQL_CMD_SEND;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return status;
+}
+
+/*
+ * \syncpipeline -- send a sync message to an active pipeline
+ */
+static backslashResult
+exec_command_syncpipeline(PsqlScanState scan_state, bool active_branch)
+{
+ backslashResult status = PSQL_CMD_SKIP_LINE;
+
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_SEND_PIPELINE_SYNC;
+ status = PSQL_CMD_SEND;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return status;
+}
+
+/*
+ * \endpipeline -- end pipeline mode
+ */
+static backslashResult
+exec_command_endpipeline(PsqlScanState scan_state, bool active_branch)
+{
+ backslashResult status = PSQL_CMD_SKIP_LINE;
+
+ if (active_branch)
+ {
+ pset.send_mode = PSQL_SEND_END_PIPELINE_MODE;
+ status = PSQL_CMD_SEND;
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return status;
+}
+
/*
* \t -- turn off table headers and row count
*/
diff --git a/src/bin/psql/common.c b/src/bin/psql/common.c
index f1a5291c13b..b7fe555d56c 100644
--- a/src/bin/psql/common.c
+++ b/src/bin/psql/common.c
@@ -121,6 +121,18 @@ CloseGOutput(FILE *gfile_fout, bool is_pipe)
}
}
+/*
+ * Reset pset pipeline state
+ */
+static void
+pipelineReset(void)
+{
+ pset.piped_syncs = 0;
+ pset.piped_commands = 0;
+ pset.available_results = 0;
+ pset.requested_results = 0;
+}
+
/*
* setQFout
* -- handler for -o command line option and \o command
@@ -354,6 +366,7 @@ CheckConnection(void)
fprintf(stderr, _("The connection to the server was lost. Attempting reset: "));
PQreset(pset.db);
+ pipelineReset();
OK = ConnectionUp();
if (!OK)
{
@@ -415,10 +428,12 @@ AcceptResult(const PGresult *result, bool show_error)
case PGRES_EMPTY_QUERY:
case PGRES_COPY_IN:
case PGRES_COPY_OUT:
+ case PGRES_PIPELINE_SYNC:
/* Fine, do nothing */
OK = true;
break;
+ case PGRES_PIPELINE_ABORTED:
case PGRES_BAD_RESPONSE:
case PGRES_NONFATAL_ERROR:
case PGRES_FATAL_ERROR:
@@ -1050,6 +1065,7 @@ PrintQueryResult(PGresult *result, bool last,
success = true;
break;
+ case PGRES_PIPELINE_ABORTED:
case PGRES_BAD_RESPONSE:
case PGRES_NONFATAL_ERROR:
case PGRES_FATAL_ERROR:
@@ -1418,6 +1434,63 @@ DescribeQuery(const char *query, double *elapsed_msec)
return OK;
}
+/*
+ * Read and discard all results in an aborted pipeline.
+ *
+ * If a synchronisation point is found, we can stop discarding results as
+ * pipeline will switch back to an OK state. If no synchronisation point
+ * is available, we need to stop when there's no more pending results,
+ * otherwise, calling PQgetResult will block.
+ */
+static PGresult *
+discardAbortedPipelineResults(void)
+{
+ for (;;)
+ {
+ PGresult *res = PQgetResult(pset.db);
+ ExecStatusType result_status = PQresultStatus(res);
+
+ if (result_status == PGRES_PIPELINE_SYNC)
+ {
+ /*
+ * Found a synchronisation point. Decrementing the sync counter
+ * will be done by the caller
+ */
+ return res;
+ }
+ else if (res == NULL)
+ {
+ /* A query was processed, decrement the counters */
+ Assert(pset.available_results > 0);
+ Assert(pset.requested_results > 0);
+ pset.available_results--;
+ pset.requested_results--;
+ }
+
+ if (pset.requested_results == 0)
+ {
+ /* We've read all requested results, exit */
+ return res;
+ }
+
+ if (pset.available_results == 0 && pset.piped_syncs == 0)
+ {
+ /*
+ * There's no more results to get and there's no synchronisation
+ * point to stop at. This will leave the pipeline in an aborted
+ * state.
+ */
+ return res;
+ }
+
+ /*
+ * An aborted pipeline will have either NULL results or results in an
+ * PGRES_PIPELINE_ABORTED status
+ */
+ Assert(res == NULL || result_status == PGRES_PIPELINE_ABORTED);
+ PQclear(res);
+ }
+}
/*
* ExecQueryAndProcessResults: utility function for use by SendQuery()
@@ -1451,6 +1524,7 @@ ExecQueryAndProcessResults(const char *query,
bool timing = pset.timing;
bool success = false;
bool return_early = false;
+ bool end_pipeline = false;
instr_time before,
after;
PGresult *result;
@@ -1466,9 +1540,13 @@ ExecQueryAndProcessResults(const char *query,
{
case PSQL_SEND_EXTENDED_CLOSE:
success = PQsendClosePrepared(pset.db, pset.stmtName);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ pset.piped_commands++;
break;
case PSQL_SEND_EXTENDED_PARSE:
success = PQsendPrepare(pset.db, pset.stmtName, query, 0, NULL);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ pset.piped_commands++;
break;
case PSQL_SEND_EXTENDED_QUERY_PARAMS:
Assert(pset.stmtName == NULL);
@@ -1476,6 +1554,8 @@ ExecQueryAndProcessResults(const char *query,
pset.bind_nparams, NULL,
(const char *const *) pset.bind_params,
NULL, NULL, 0);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ pset.piped_commands++;
break;
case PSQL_SEND_EXTENDED_QUERY_PREPARED:
Assert(pset.stmtName != NULL);
@@ -1483,6 +1563,83 @@ ExecQueryAndProcessResults(const char *query,
pset.bind_nparams,
(const char *const *) pset.bind_params,
NULL, NULL, 0);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ pset.piped_commands++;
+ break;
+ case PSQL_SEND_START_PIPELINE_MODE:
+ success = PQenterPipelineMode(pset.db);
+ break;
+ case PSQL_SEND_END_PIPELINE_MODE:
+ success = PQpipelineSync(pset.db);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ /*
+ * End of the pipeline, all queued commands need to be
+ * processed
+ */
+ end_pipeline = true;
+ pset.piped_syncs++;
+
+ /*
+ * The server will send a ReadyForQuery after a Sync is
+ * processed, flushing all results to the client
+ */
+ pset.available_results += pset.piped_commands;
+ pset.piped_commands = 0;
+ /* We want to read all results */
+ pset.requested_results = pset.available_results + pset.piped_syncs;
+ }
+ break;
+ case PSQL_SEND_PIPELINE_SYNC:
+ success = PQsendPipelineSync(pset.db);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pset.piped_syncs++;
+
+ /*
+ * The server will send a ReadyForQuery after a Sync is
+ * processed, flushing all results to the client
+ */
+ pset.available_results += pset.piped_commands;
+ pset.piped_commands = 0;
+ }
+ break;
+ case PSQL_SEND_FLUSH:
+ success = PQflush(pset.db);
+ break;
+ case PSQL_SEND_FLUSH_REQUEST:
+ success = PQsendFlushRequest(pset.db);
+ if (success && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ /*
+ * With the flush request, all piped commands are pushed and
+ * the server will forcefully flush the results to the client,
+ * making them available
+ */
+ pset.available_results += pset.piped_commands;
+ pset.piped_commands = 0;
+ }
+ break;
+ case PSQL_SEND_GET_RESULTS:
+ if (pset.available_results == 0 && pset.piped_syncs == 0)
+ {
+ /*
+ * If no sync or flush request were sent, PQgetResult will
+ * block. Forbid the call to \getresults to avoid staying
+ * stuck
+ */
+ pg_log_info("No pending results to get");
+ success = false;
+ pset.requested_results = 0;
+ }
+ else
+ {
+ success = true;
+ /* Cap requested_results to the maximum known results */
+ if (pset.requested_results == 0 ||
+ pset.requested_results > (pset.available_results + pset.piped_syncs))
+ pset.requested_results = pset.available_results + pset.piped_syncs;
+ }
break;
case PSQL_SEND_QUERY:
success = PQsendQuery(pset.db, query);
@@ -1501,6 +1658,16 @@ ExecQueryAndProcessResults(const char *query,
return -1;
}
+ if (pset.requested_results == 0 && !end_pipeline &&
+ PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ /*
+ * We're in a pipeline and haven't reached the pipeline end or there
+ * was no request to read pipeline results, exit.
+ */
+ return 1;
+ }
+
/*
* Fetch the result in chunks if FETCH_COUNT is set, except when:
*
@@ -1548,7 +1715,7 @@ ExecQueryAndProcessResults(const char *query,
{
ExecStatusType result_status;
bool is_chunked_result = false;
- PGresult *next_result;
+ PGresult *next_result = NULL;
bool last;
if (!AcceptResult(result, false))
@@ -1571,6 +1738,9 @@ ExecQueryAndProcessResults(const char *query,
ClearOrSaveResult(result);
success = false;
+ if (result_status == PGRES_PIPELINE_ABORTED)
+ pg_log_info("Pipeline aborted, command didn't run");
+
/*
* switch to next result
*/
@@ -1585,6 +1755,22 @@ ExecQueryAndProcessResults(const char *query,
* ignore manually.
*/
result = NULL;
+ else if ((end_pipeline || pset.requested_results > 0)
+ && PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+
+ /*
+ * We have an error within a pipeline. All commands are
+ * aborted until the next synchronisation point. We need to
+ * consume all results until this synchronisation point, or
+ * stop when there's no more result to discard
+ *
+ * Checking pipeline status is necessary in case the
+ * connection was reset. The new connection isn't in any kind
+ * of pipeline state and thus has no result to discard
+ */
+ result = discardAbortedPipelineResults();
+ }
else
result = PQgetResult(pset.db);
@@ -1771,12 +1957,66 @@ ExecQueryAndProcessResults(const char *query,
}
}
+ if (result_status == PGRES_PIPELINE_SYNC)
+ {
+ Assert(pset.piped_syncs > 0);
+
+ /*
+ * We have a sync response, decrease the sync and
+ * requested_results counters
+ */
+ pset.piped_syncs--;
+ pset.requested_results--;
+
+ /*
+ * After a synchronisation point, reset success state to print
+ * possible successful results that will be processed after this
+ */
+ success = true;
+
+ /*
+ * If all syncs were processed and pipeline end was requested,
+ * exit pipeline mode
+ */
+ if (end_pipeline && pset.piped_syncs == 0)
+ success &= PQexitPipelineMode(pset.db);
+ }
+ else if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF &&
+ result_status != PGRES_PIPELINE_SYNC)
+ {
+ /*
+ * We are in a pipeline and have a non sync response, decrease the
+ * results counters
+ */
+ pset.available_results--;
+ pset.requested_results--;
+ }
+
/*
* Check PQgetResult() again. In the typical case of a single-command
* string, it will return NULL. Otherwise, we'll have other results
* to process. We need to do that to check whether this is the last.
*/
- next_result = PQgetResult(pset.db);
+ if (PQpipelineStatus(pset.db) == PQ_PIPELINE_OFF)
+ next_result = PQgetResult(pset.db);
+ else
+ {
+ /*
+ * In pipeline mode, a NULL result indicates the end of the
+ * current query being processed. Call PQgetResult to consume this
+ * NULL
+ */
+ if (result_status != PGRES_PIPELINE_SYNC)
+ {
+ next_result = PQgetResult(pset.db);
+ Assert(next_result == NULL);
+ }
+
+ /* We can now get the next result in the pipeline */
+ if (pset.requested_results > 0)
+ next_result = PQgetResult(pset.db);
+ }
+
last = (next_result == NULL);
/*
@@ -1798,8 +2038,12 @@ ExecQueryAndProcessResults(const char *query,
*elapsed_msec = INSTR_TIME_GET_MILLISEC(after);
}
- /* this may or may not print something depending on settings */
- if (result != NULL)
+ /*
+ * This may or may not print something depending on settings. A
+ * pipeline sync will have a non null result but doesn't have anything
+ * to print, thus ignore them
+ */
+ if (result != NULL && result_status != PGRES_PIPELINE_SYNC)
{
/*
* If results need to be printed into the file specified by \g,
@@ -1825,9 +2069,15 @@ ExecQueryAndProcessResults(const char *query,
ClearOrSaveResult(result);
result = next_result;
- if (cancel_pressed)
+ if (cancel_pressed && PQpipelineStatus(pset.db) == PQ_PIPELINE_OFF)
{
- /* drop this next result, as well as any others not yet read */
+ /*
+ * Outside of a pipeline, drop this next result, as well as any
+ * others not yet read
+ *
+ * Within a pipeline, we can let the outer loop handle this as an
+ * aborted pipeline, which will discard all results
+ */
ClearOrSaveResult(result);
ClearOrSaveAllResults();
break;
@@ -1837,6 +2087,17 @@ ExecQueryAndProcessResults(const char *query,
/* close \g file if we opened it */
CloseGOutput(gfile_fout, gfile_is_pipe);
+ if (end_pipeline)
+ {
+ /* After a pipeline is processed, pipeline piped_syncs should be 0 */
+ Assert(pset.piped_syncs == 0);
+ /* all commands were processed */
+ Assert(pset.piped_commands == 0);
+ /* and all results were read */
+ Assert(pset.available_results == 0);
+ }
+ Assert(pset.requested_results == 0);
+
/* may need this to recover from conn loss during COPY */
if (!CheckConnection())
return -1;
@@ -2297,6 +2558,12 @@ clean_extended_state(void)
pset.bind_params = NULL;
break;
case PSQL_SEND_QUERY:
+ case PSQL_SEND_START_PIPELINE_MODE: /* \startpipeline */
+ case PSQL_SEND_END_PIPELINE_MODE: /* \endpipeline */
+ case PSQL_SEND_PIPELINE_SYNC: /* \syncpipeline */
+ case PSQL_SEND_FLUSH: /* \flush */
+ case PSQL_SEND_GET_RESULTS: /* \getresults */
+ case PSQL_SEND_FLUSH_REQUEST: /* \flushrequest */
break;
}
diff --git a/src/bin/psql/help.c b/src/bin/psql/help.c
index da8e1ade5df..714b8619233 100644
--- a/src/bin/psql/help.c
+++ b/src/bin/psql/help.c
@@ -167,15 +167,22 @@ slashUsage(unsigned short int pager)
HELP0(" \\close STMT_NAME close an existing prepared statement\n");
HELP0(" \\copyright show PostgreSQL usage and distribution terms\n");
HELP0(" \\crosstabview [COLUMNS] execute query and display result in crosstab\n");
+ HELP0(" \\endpipeline exit pipeline mode\n");
HELP0(" \\errverbose show most recent error message at maximum verbosity\n");
+ HELP0(" \\flush push unsent data to the server\n");
+ HELP0(" \\flushrequest send a flushrequest command\n");
HELP0(" \\g [(OPTIONS)] [FILE] execute query (and send result to file or |pipe);\n"
" \\g with no arguments is equivalent to a semicolon\n");
HELP0(" \\gdesc describe result of query, without executing it\n");
+ HELP0(" \\getresults [NUM_RES] read NUM_RES pending results. All pending results are\n"
+ " read if no argument is provided\n");
HELP0(" \\gexec execute query, then execute each value in its result\n");
HELP0(" \\gset [PREFIX] execute query and store result in psql variables\n");
HELP0(" \\gx [(OPTIONS)] [FILE] as \\g, but forces expanded output mode\n");
HELP0(" \\parse STMT_NAME create a prepared statement\n");
HELP0(" \\q quit psql\n");
+ HELP0(" \\startpipeline enter pipeline mode\n");
+ HELP0(" \\syncpipeline add a synchronisation point to an ongoing pipeline\n");
HELP0(" \\watch [[i=]SEC] [c=N] [m=MIN]\n"
" execute query every SEC seconds, up to N times,\n"
" stop if less than MIN rows are returned\n");
diff --git a/src/bin/psql/settings.h b/src/bin/psql/settings.h
index 2a8fe12eb55..022a6e9183e 100644
--- a/src/bin/psql/settings.h
+++ b/src/bin/psql/settings.h
@@ -69,6 +69,12 @@ typedef enum
PSQL_SEND_EXTENDED_PARSE,
PSQL_SEND_EXTENDED_QUERY_PARAMS,
PSQL_SEND_EXTENDED_QUERY_PREPARED,
+ PSQL_SEND_PIPELINE_SYNC,
+ PSQL_SEND_START_PIPELINE_MODE,
+ PSQL_SEND_END_PIPELINE_MODE,
+ PSQL_SEND_FLUSH,
+ PSQL_SEND_FLUSH_REQUEST,
+ PSQL_SEND_GET_RESULTS,
} PSQL_SEND_MODE;
typedef enum
@@ -111,6 +117,12 @@ typedef struct _psqlSettings
char **bind_params; /* parameters for extended query protocol call */
char *stmtName; /* prepared statement name used for extended
* query protocol commands */
+ int piped_commands; /* number of piped commands */
+ int piped_syncs; /* number of piped syncs */
+ int available_results; /* number of results available to get */
+ int requested_results; /* number of requested results, include
+ * sync messages. Used to read a limited
+ * subset of the available_results */
bool crosstab_flag; /* one-shot request to crosstab result */
char *ctv_args[4]; /* \crosstabview arguments */
diff --git a/src/bin/psql/tab-complete.in.c b/src/bin/psql/tab-complete.in.c
index eb8bc128720..8432be641ac 100644
--- a/src/bin/psql/tab-complete.in.c
+++ b/src/bin/psql/tab-complete.in.c
@@ -1885,9 +1885,9 @@ psql_completion(const char *text, int start, int end)
"\\drds", "\\drg", "\\dRs", "\\dRp", "\\ds",
"\\dt", "\\dT", "\\dv", "\\du", "\\dx", "\\dX", "\\dy",
"\\echo", "\\edit", "\\ef", "\\elif", "\\else", "\\encoding",
- "\\endif", "\\errverbose", "\\ev",
- "\\f",
- "\\g", "\\gdesc", "\\getenv", "\\gexec", "\\gset", "\\gx",
+ "\\endif", "\\endpipeline", "\\errverbose", "\\ev",
+ "\\f", "\\flush", "\\flushrequest",
+ "\\g", "\\gdesc", "\\getenv", "\\getresults", "\\gexec", "\\gset", "\\gx",
"\\help", "\\html",
"\\if", "\\include", "\\include_relative", "\\ir",
"\\list", "\\lo_import", "\\lo_export", "\\lo_list", "\\lo_unlink",
@@ -1895,7 +1895,7 @@ psql_completion(const char *text, int start, int end)
"\\parse", "\\password", "\\print", "\\prompt", "\\pset",
"\\qecho", "\\quit",
"\\reset",
- "\\s", "\\set", "\\setenv", "\\sf", "\\sv",
+ "\\s", "\\set", "\\setenv", "\\sf", "\\startpipeline", "\\sv", "\\syncpipeline",
"\\t", "\\T", "\\timing",
"\\unset",
"\\x",
diff --git a/src/test/regress/expected/psql.out b/src/test/regress/expected/psql.out
index f9db4032e1f..6543e90de75 100644
--- a/src/test/regress/expected/psql.out
+++ b/src/test/regress/expected/psql.out
@@ -4678,11 +4678,15 @@ bar 'bar' "bar"
\echo arg1 arg2 arg3 arg4 arg5
\echo arg1
\encoding arg1
+ \endpipeline
\errverbose
\f arg1
+ \flush
+ \flushrequest
\g arg1
\gx arg1
\gexec
+ \getresults
SELECT 1 AS one \gset
\h
\?
@@ -4706,6 +4710,8 @@ invalid command \lo
\setenv arg1 arg2
\sf whole_line
\sv whole_line
+ \startpipeline
+ \syncpipeline
\t arg1
\T arg1
\timing arg1
diff --git a/src/test/regress/expected/psql_pipeline.out b/src/test/regress/expected/psql_pipeline.out
new file mode 100644
index 00000000000..1da63e43558
--- /dev/null
+++ b/src/test/regress/expected/psql_pipeline.out
@@ -0,0 +1,684 @@
+--
+-- Tests using psql pipelining
+--
+CREATE TABLE psql_pipeline(a INTEGER PRIMARY KEY, s TEXT);
+-- single query
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+-- multiple queries
+\startpipeline
+SELECT $1 \bind 'val1' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val2 | val3
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val2 | val3
+(1 row)
+
+-- Test \flush
+\startpipeline
+\flush
+SELECT $1 \bind 'val1' \g
+\flush
+SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val2 | val3
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val2 | val3
+(1 row)
+
+-- send multiple syncs
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\syncpipeline
+\syncpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\syncpipeline
+SELECT $1, $2 \bind 'val4' 'val5' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val2 | val3
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val4 | val5
+(1 row)
+
+-- startpipeline shouldn't have any effect if already in a pipeline
+\startpipeline
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+-- Convert an implicit tx block to an explicit tx block
+\startpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 2 \g
+ROLLBACK \bind \g
+\endpipeline
+-- Multiple explicit transactions
+\startpipeline
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+ROLLBACK \bind \g
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+COMMIT \bind \g
+\endpipeline
+-- COPY FROM STDIN
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+-- COPY FROM STDIN with \flushrequest + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\flushrequest
+\getresults
+ ?column?
+----------
+ val1
+(1 row)
+
+message type 0x5a arrived from server while idle
+\endpipeline
+-- COPY FROM STDIN with \syncpipeline + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\syncpipeline
+\getresults
+ ?column?
+----------
+ val1
+(1 row)
+
+\endpipeline
+-- COPY TO STDOUT
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+1 \N
+2 test2
+3 test3
+4 test4
+-- COPY TO STDOUT with \flushrequest + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\flushrequest
+\getresults
+ ?column?
+----------
+ val1
+(1 row)
+
+1 \N
+2 test2
+3 test3
+4 test4
+\endpipeline
+-- COPY TO STDOUT with \syncpipeline + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\syncpipeline
+\getresults
+ ?column?
+----------
+ val1
+(1 row)
+
+1 \N
+2 test2
+3 test3
+4 test4
+\endpipeline
+-- Use \parse and \bind_named
+\startpipeline
+SELECT $1 \parse ''
+SELECT $1, $2 \parse ''
+SELECT $2 \parse pipeline_1
+\bind_named '' 1 2 \g
+\bind_named pipeline_1 2 \g
+\endpipeline
+ERROR: could not determine data type of parameter $1
+-- \getresults displays all results preceding a \flushrequest
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+\flushrequest
+\getresults
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+\endpipeline
+-- \getresults displays all results preceding a sync
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+\syncpipeline
+\getresults
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+\endpipeline
+-- \getresults immediately returns if there's no result to fetch
+\startpipeline
+\getresults
+No pending results to get
+SELECT $1 \bind 2 \g
+\getresults
+No pending results to get
+\flushrequest
+\endpipeline
+ ?column?
+----------
+ 2
+(1 row)
+
+\getresults
+No pending results to get
+-- \getresults only fetch results preceding a flushrequest
+\startpipeline
+SELECT $1 \bind 2 \g
+\flushrequest
+SELECT $1 \bind 2 \g
+\getresults
+ ?column?
+----------
+ 2
+(1 row)
+
+\endpipeline
+ ?column?
+----------
+ 2
+(1 row)
+
+-- \getresults only fetch results preceding a sync message
+\startpipeline
+SELECT $1 \bind 2 \g
+\syncpipeline
+SELECT $1 \bind 2 \g
+\getresults
+ ?column?
+----------
+ 2
+(1 row)
+
+\endpipeline
+ ?column?
+----------
+ 2
+(1 row)
+
+-- use pipeline with chunked results with both \getresults and \endpipeline
+\startpipeline
+\set FETCH_COUNT 10
+SELECT $1 \bind 2 \g
+\flushrequest
+\getresults
+ ?column?
+----------
+ 2
+(1 row)
+
+SELECT $1 \bind 2 \g
+\endpipeline
+ ?column?
+----------
+ 2
+(1 row)
+
+\unset FETCH_COUNT
+-- \getresults with specific number of requested results
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+SELECT $1 \bind 3 \g
+\syncpipeline
+\getresults 1
+ ?column?
+----------
+ 1
+(1 row)
+
+SELECT $1 \bind 4 \g
+\getresults 3
+ ?column?
+----------
+ 2
+(1 row)
+
+ ?column?
+----------
+ 3
+(1 row)
+
+\endpipeline
+ ?column?
+----------
+ 4
+(1 row)
+
+-- \syncpipeline count as one command to fetch for \getresults
+\startpipeline
+\syncpipeline
+\syncpipeline
+SELECT $1 \bind 1 \g
+\flushrequest
+\getresults 2
+\getresults 1
+ ?column?
+----------
+ 1
+(1 row)
+
+\endpipeline
+-- \getresults 0 should get all results
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+SELECT $1 \bind 3 \g
+\syncpipeline
+\getresults 0
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+ ?column?
+----------
+ 3
+(1 row)
+
+\endpipeline
+-- pipelining errors
+-- endpipeline outside of pipeline should fail
+\endpipeline
+cannot send pipeline when not in pipeline mode
+-- Query using simple protocol should not be sent and should leave the pipeline usable
+\startpipeline
+SELECT 1;
+PQsendQuery not allowed in pipeline mode
+SELECT $1 \bind 'val1' \g
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+-- After an aborted pipeline, commands after a sync should be displayed
+\startpipeline
+SELECT $1 \bind \g
+\syncpipeline
+SELECT $1 \bind 1 \g
+\endpipeline
+ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
+ ?column?
+----------
+ 1
+(1 row)
+
+-- Incorrect number of parameters, the pipeline will be aborted and following queries won't be executed
+\startpipeline
+SELECT \bind 'val1' \g
+SELECT $1 \bind 'val1' \g
+\endpipeline
+ERROR: bind message supplies 1 parameters, but prepared statement "" requires 0
+-- An explicit transaction with an error needs to be rollbacked after the pipeline
+\startpipeline
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+ROLLBACK \bind \g
+\endpipeline
+ERROR: duplicate key value violates unique constraint "psql_pipeline_pkey"
+DETAIL: Key (a)=(1) already exists.
+ROLLBACK;
+-- \watch sends a simple query which won't be allowed within a pipeline
+\startpipeline
+SELECT \bind \g
+\watch 1
+PQsendQuery not allowed in pipeline mode
+
+\endpipeline
+--
+(1 row)
+
+-- \gdesc should fail as synchronous commands are not allowed in pipeline, pipeline should still be usable
+\startpipeline
+SELECT $1 \bind 1 \gdesc
+synchronous command execution functions are not allowed in pipeline mode
+SELECT $1 \bind 1 \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+-- \gset is not allowed, pipeline should still be usable
+\startpipeline
+SELECT $1 as i, $2 as j \parse ''
+SELECT $1 as k, $2 as l \parse 'second'
+\bind_named '' 1 2 \gset
+\gset not allowed in pipeline mode
+\bind_named second 1 2 \gset pref02_ \echo :pref02_i :pref02_j
+\gset not allowed in pipeline mode
+\bind_named '' 1 2 \g
+\endpipeline
+ i | j
+---+---
+ 1 | 2
+(1 row)
+
+-- \gx is not allowed, pipeline should still be usable
+\startpipeline
+SELECT $1 \bind 1 \gx
+\gx not allowed in pipeline mode
+\reset
+SELECT $1 \bind 1 \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+-- \gx warning should be emitted in an aborted pipeline
+\startpipeline
+SELECT $1 \bind \g
+\flushrequest
+\getresults
+ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
+SELECT $1 \bind 1 \gx
+\gx not allowed in pipeline mode
+\endpipeline
+-- \gexec is not allowed, pipeline should still be usable
+\startpipeline
+SELECT 'INSERT INTO psql_pipeline(a) SELECT generate_series(1, 10)' \parse 'insert_stmt'
+\bind_named insert_stmt \gexec
+\gexec not allowed in pipeline mode
+\bind_named insert_stmt \g
+SELECT COUNT(*) FROM psql_pipeline \bind \g
+\endpipeline
+ ?column?
+------------------------------------------------------------
+ INSERT INTO psql_pipeline(a) SELECT generate_series(1, 10)
+(1 row)
+
+ count
+-------
+ 4
+(1 row)
+
+-- After an error, pipeline is aborted and requires a sync to be reusable
+\startpipeline
+SELECT $1 \bind \g
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+\flushrequest
+\getresults
+ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
+-- Pipeline is aborted
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+-- Sync allows pipeline to recover
+\syncpipeline
+\getresults
+Pipeline aborted, command didn't run
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+\flushrequest
+\getresults
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 1
+(1 row)
+
+\endpipeline
+-- In an aborted pipeline, \getresults 1 aborted commands one at a time
+\startpipeline
+SELECT $1 \bind \g
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\syncpipeline
+\getresults 1
+ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
+\getresults 1
+Pipeline aborted, command didn't run
+\getresults 1
+Pipeline aborted, command didn't run
+\getresults 1
+Pipeline aborted, command didn't run
+\getresults 1
+\endpipeline
+-- Test chunked results with an aborted pipeline
+\startpipeline
+\set FETCH_COUNT 10
+SELECT $1 \bind \g
+\flushrequest
+\getresults
+ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
+SELECT $1 \bind \g
+\endpipeline
+fetching results in chunked mode failed
+Pipeline aborted, command didn't run
+\unset FETCH_COUNT
+-- \getresults returns an error when an incorrect number is provided
+\startpipeline
+\getresults -1
+\getresults: invalid number of requested results
+\endpipeline
+-- \getresults when there's no result shouldn't impact the following query
+\getresults 1
+No pending results to get
+select 1;
+ ?column?
+----------
+ 1
+(1 row)
+
+-- pipelining and transaction block behaviour
+-- set local will issue a warning when modifying a GUC outside of a transaction block
+-- The change will still be valid as a pipeline runs within an implicit transaction block
+-- Sending a sync will commit the implicit transaction block. The first command after a sync
+-- won't be seen as belonging to a pipeline.
+\startpipeline
+SET LOCAL statement_timeout='1h' \bind \g
+SHOW statement_timeout \bind \g
+\syncpipeline
+SHOW statement_timeout \bind \g
+SET LOCAL statement_timeout='2h' \bind \g
+SHOW statement_timeout \bind \g
+\endpipeline
+WARNING: SET LOCAL can only be used in transaction blocks
+ statement_timeout
+-------------------
+ 1h
+(1 row)
+
+ statement_timeout
+-------------------
+ 0
+(1 row)
+
+ statement_timeout
+-------------------
+ 2h
+(1 row)
+
+-- Reindex concurrently is forbidden in the middle of a pipeline
+\startpipeline
+SELECT $1 \bind 1 \g
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ERROR: REINDEX CONCURRENTLY cannot run inside a transaction block
+-- Reindex concurrently will work if it's the first command of a pipeline
+\startpipeline
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ ?column?
+----------
+ 2
+(1 row)
+
+-- subtransactions are not allowed in pipeline mode
+\startpipeline
+SAVEPOINT a \bind \g
+SELECT $1 \bind 1 \g
+ROLLBACK TO SAVEPOINT a \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ERROR: SAVEPOINT can only be used in transaction blocks
+-- Lock command will fail as first pipeline command is not seen as a transaction block
+\startpipeline
+LOCK psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ERROR: LOCK TABLE can only be used in transaction blocks
+-- Lock command will succeed after the first command as pipeline will be seen as an implicit transaction block
+\startpipeline
+SELECT $1 \bind 1 \g
+LOCK psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+-- Vacuum command will work as the first command
+\startpipeline
+VACUUM psql_pipeline \bind \g
+\endpipeline
+-- Vacuum command will fail within pipeline implicit transaction
+\startpipeline
+SELECT 1 \bind \g
+VACUUM psql_pipeline \bind \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ERROR: VACUUM cannot run inside a transaction block
+-- Vacuum command will work after a sync
+\startpipeline
+SELECT 1 \bind \g
+\syncpipeline
+VACUUM psql_pipeline \bind \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+-- Error messages are currently accumulating and will be repeated
+\startpipeline
+SELECT 1 \bind \g
+SELECT 1;
+PQsendQuery not allowed in pipeline mode
+SELECT 1;
+PQsendQuery not allowed in pipeline mode
+PQsendQuery not allowed in pipeline mode
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule
index e63ee2cf2bb..37b6d21e1f9 100644
--- a/src/test/regress/parallel_schedule
+++ b/src/test/regress/parallel_schedule
@@ -81,7 +81,7 @@ test: brin_bloom brin_multi
test: create_table_like alter_generic alter_operator misc async dbsize merge misc_functions sysviews tsrf tid tidscan tidrangescan collate.utf8 collate.icu.utf8 incremental_sort create_role without_overlaps generated_virtual
# collate.linux.utf8 and collate.icu.utf8 tests cannot be run in parallel with each other
-test: rules psql psql_crosstab amutils stats_ext collate.linux.utf8 collate.windows.win1252
+test: rules psql psql_crosstab psql_pipeline amutils stats_ext collate.linux.utf8 collate.windows.win1252
# ----------
# Run these alone so they don't run out of parallel workers
diff --git a/src/test/regress/sql/psql.sql b/src/test/regress/sql/psql.sql
index c58308ce14f..97d1be3aac3 100644
--- a/src/test/regress/sql/psql.sql
+++ b/src/test/regress/sql/psql.sql
@@ -1047,11 +1047,15 @@ select \if false \\ (bogus \else \\ 42 \endif \\ forty_two;
\echo arg1 arg2 arg3 arg4 arg5
\echo arg1
\encoding arg1
+ \endpipeline
\errverbose
\f arg1
+ \flush
+ \flushrequest
\g arg1
\gx arg1
\gexec
+ \getresults
SELECT 1 AS one \gset
\h
\?
@@ -1074,6 +1078,8 @@ select \if false \\ (bogus \else \\ 42 \endif \\ forty_two;
\setenv arg1 arg2
\sf whole_line
\sv whole_line
+ \startpipeline
+ \syncpipeline
\t arg1
\T arg1
\timing arg1
diff --git a/src/test/regress/sql/psql_pipeline.sql b/src/test/regress/sql/psql_pipeline.sql
new file mode 100644
index 00000000000..bea48560a93
--- /dev/null
+++ b/src/test/regress/sql/psql_pipeline.sql
@@ -0,0 +1,408 @@
+--
+-- Tests using psql pipelining
+--
+
+CREATE TABLE psql_pipeline(a INTEGER PRIMARY KEY, s TEXT);
+
+-- single query
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\endpipeline
+
+-- multiple queries
+\startpipeline
+SELECT $1 \bind 'val1' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\endpipeline
+
+-- Test \flush
+\startpipeline
+\flush
+SELECT $1 \bind 'val1' \g
+\flush
+SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\endpipeline
+
+-- send multiple syncs
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\syncpipeline
+\syncpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \g
+\syncpipeline
+SELECT $1, $2 \bind 'val4' 'val5' \g
+\endpipeline
+
+-- startpipeline shouldn't have any effect if already in a pipeline
+\startpipeline
+\startpipeline
+SELECT $1 \bind 'val1' \g
+\endpipeline
+
+-- Convert an implicit tx block to an explicit tx block
+\startpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 2 \g
+ROLLBACK \bind \g
+\endpipeline
+
+-- Multiple explicit transactions
+\startpipeline
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+ROLLBACK \bind \g
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+COMMIT \bind \g
+\endpipeline
+
+-- COPY FROM STDIN
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\endpipeline
+2 test2
+\.
+
+-- COPY FROM STDIN with \flushrequest + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\flushrequest
+\getresults
+3 test3
+\.
+\endpipeline
+
+-- COPY FROM STDIN with \syncpipeline + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+COPY psql_pipeline FROM STDIN \bind \g
+\syncpipeline
+\getresults
+4 test4
+\.
+\endpipeline
+
+-- COPY TO STDOUT
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\endpipeline
+
+-- COPY TO STDOUT with \flushrequest + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\flushrequest
+\getresults
+\endpipeline
+
+-- COPY TO STDOUT with \syncpipeline + \getresults
+\startpipeline
+SELECT $1 \bind 'val1' \g
+copy psql_pipeline TO STDOUT \bind \g
+\syncpipeline
+\getresults
+\endpipeline
+
+-- Use \parse and \bind_named
+\startpipeline
+SELECT $1 \parse ''
+SELECT $1, $2 \parse ''
+SELECT $2 \parse pipeline_1
+\bind_named '' 1 2 \g
+\bind_named pipeline_1 2 \g
+\endpipeline
+
+-- \getresults displays all results preceding a \flushrequest
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+\flushrequest
+\getresults
+\endpipeline
+
+-- \getresults displays all results preceding a sync
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+\syncpipeline
+\getresults
+\endpipeline
+
+-- \getresults immediately returns if there's no result to fetch
+\startpipeline
+\getresults
+SELECT $1 \bind 2 \g
+\getresults
+\flushrequest
+\endpipeline
+\getresults
+
+-- \getresults only fetch results preceding a flushrequest
+\startpipeline
+SELECT $1 \bind 2 \g
+\flushrequest
+SELECT $1 \bind 2 \g
+\getresults
+\endpipeline
+
+-- \getresults only fetch results preceding a sync message
+\startpipeline
+SELECT $1 \bind 2 \g
+\syncpipeline
+SELECT $1 \bind 2 \g
+\getresults
+\endpipeline
+
+-- use pipeline with chunked results with both \getresults and \endpipeline
+\startpipeline
+\set FETCH_COUNT 10
+SELECT $1 \bind 2 \g
+\flushrequest
+\getresults
+SELECT $1 \bind 2 \g
+\endpipeline
+\unset FETCH_COUNT
+
+-- \getresults with specific number of requested results
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+SELECT $1 \bind 3 \g
+\syncpipeline
+\getresults 1
+SELECT $1 \bind 4 \g
+\getresults 3
+\endpipeline
+
+-- \syncpipeline count as one command to fetch for \getresults
+\startpipeline
+\syncpipeline
+\syncpipeline
+SELECT $1 \bind 1 \g
+\flushrequest
+\getresults 2
+\getresults 1
+\endpipeline
+
+-- \getresults 0 should get all results
+\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 2 \g
+SELECT $1 \bind 3 \g
+\syncpipeline
+\getresults 0
+\endpipeline
+
+-- pipelining errors
+
+-- endpipeline outside of pipeline should fail
+\endpipeline
+
+-- Query using simple protocol should not be sent and should leave the pipeline usable
+\startpipeline
+SELECT 1;
+SELECT $1 \bind 'val1' \g
+\endpipeline
+
+-- After an aborted pipeline, commands after a sync should be displayed
+\startpipeline
+SELECT $1 \bind \g
+\syncpipeline
+SELECT $1 \bind 1 \g
+\endpipeline
+
+-- Incorrect number of parameters, the pipeline will be aborted and following queries won't be executed
+\startpipeline
+SELECT \bind 'val1' \g
+SELECT $1 \bind 'val1' \g
+\endpipeline
+
+-- An explicit transaction with an error needs to be rollbacked after the pipeline
+\startpipeline
+BEGIN \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
+ROLLBACK \bind \g
+\endpipeline
+ROLLBACK;
+
+-- \watch sends a simple query which won't be allowed within a pipeline
+\startpipeline
+SELECT \bind \g
+\watch 1
+\endpipeline
+
+-- \gdesc should fail as synchronous commands are not allowed in pipeline, pipeline should still be usable
+\startpipeline
+SELECT $1 \bind 1 \gdesc
+SELECT $1 \bind 1 \g
+\endpipeline
+
+-- \gset is not allowed, pipeline should still be usable
+\startpipeline
+SELECT $1 as i, $2 as j \parse ''
+SELECT $1 as k, $2 as l \parse 'second'
+\bind_named '' 1 2 \gset
+\bind_named second 1 2 \gset pref02_ \echo :pref02_i :pref02_j
+\bind_named '' 1 2 \g
+\endpipeline
+
+-- \gx is not allowed, pipeline should still be usable
+\startpipeline
+SELECT $1 \bind 1 \gx
+\reset
+SELECT $1 \bind 1 \g
+\endpipeline
+
+-- \gx warning should be emitted in an aborted pipeline
+\startpipeline
+SELECT $1 \bind \g
+\flushrequest
+\getresults
+SELECT $1 \bind 1 \gx
+\endpipeline
+
+-- \gexec is not allowed, pipeline should still be usable
+\startpipeline
+SELECT 'INSERT INTO psql_pipeline(a) SELECT generate_series(1, 10)' \parse 'insert_stmt'
+\bind_named insert_stmt \gexec
+\bind_named insert_stmt \g
+SELECT COUNT(*) FROM psql_pipeline \bind \g
+\endpipeline
+
+-- After an error, pipeline is aborted and requires a sync to be reusable
+\startpipeline
+SELECT $1 \bind \g
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+\flushrequest
+\getresults
+-- Pipeline is aborted
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+-- Sync allows pipeline to recover
+\syncpipeline
+\getresults
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\close a
+\flushrequest
+\getresults
+\endpipeline
+
+-- In an aborted pipeline, \getresults 1 aborted commands one at a time
+\startpipeline
+SELECT $1 \bind \g
+SELECT $1 \bind 1 \g
+SELECT $1 \parse a
+\bind_named a 1 \g
+\syncpipeline
+\getresults 1
+\getresults 1
+\getresults 1
+\getresults 1
+\getresults 1
+\endpipeline
+
+-- Test chunked results with an aborted pipeline
+\startpipeline
+\set FETCH_COUNT 10
+SELECT $1 \bind \g
+\flushrequest
+\getresults
+SELECT $1 \bind \g
+\endpipeline
+\unset FETCH_COUNT
+
+-- \getresults returns an error when an incorrect number is provided
+\startpipeline
+\getresults -1
+\endpipeline
+
+-- \getresults when there's no result shouldn't impact the following query
+\getresults 1
+select 1;
+
+-- pipelining and transaction block behaviour
+
+-- set local will issue a warning when modifying a GUC outside of a transaction block
+-- The change will still be valid as a pipeline runs within an implicit transaction block
+-- Sending a sync will commit the implicit transaction block. The first command after a sync
+-- won't be seen as belonging to a pipeline.
+\startpipeline
+SET LOCAL statement_timeout='1h' \bind \g
+SHOW statement_timeout \bind \g
+\syncpipeline
+SHOW statement_timeout \bind \g
+SET LOCAL statement_timeout='2h' \bind \g
+SHOW statement_timeout \bind \g
+\endpipeline
+
+-- Reindex concurrently is forbidden in the middle of a pipeline
+\startpipeline
+SELECT $1 \bind 1 \g
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- Reindex concurrently will work if it's the first command of a pipeline
+\startpipeline
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- subtransactions are not allowed in pipeline mode
+\startpipeline
+SAVEPOINT a \bind \g
+SELECT $1 \bind 1 \g
+ROLLBACK TO SAVEPOINT a \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- Lock command will fail as first pipeline command is not seen as a transaction block
+\startpipeline
+LOCK psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- Lock command will succeed after the first command as pipeline will be seen as an implicit transaction block
+\startpipeline
+SELECT $1 \bind 1 \g
+LOCK psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- Vacuum command will work as the first command
+\startpipeline
+VACUUM psql_pipeline \bind \g
+\endpipeline
+
+-- Vacuum command will fail within pipeline implicit transaction
+\startpipeline
+SELECT 1 \bind \g
+VACUUM psql_pipeline \bind \g
+\endpipeline
+
+-- Vacuum command will work after a sync
+\startpipeline
+SELECT 1 \bind \g
+\syncpipeline
+VACUUM psql_pipeline \bind \g
+\endpipeline
+
+-- Error messages are currently accumulating and will be repeated
+\startpipeline
+SELECT 1 \bind \g
+SELECT 1;
+SELECT 1;
+\endpipeline
--
2.39.5 (Apple Git-154)
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-02-21 02:33 Michael Paquier <[email protected]>
parent: Anthonin Bonnefoy <[email protected]>
0 siblings, 1 reply; 41+ messages in thread
From: Michael Paquier @ 2025-02-21 02:33 UTC (permalink / raw)
To: Anthonin Bonnefoy <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; pgsql-hackers
On Thu, Feb 20, 2025 at 10:29:33AM +0100, Anthonin Bonnefoy wrote:
> Ha yeah, I forgot about the inactive branches. I've added the new
> commands and fixed the behaviour.
And I did not notice that it was as simple as forcing the status in
the routines for the new meta-commands, as we do for the existing
ones. Noted.
> This looks more like an issue on libpq's side as there's no way to
> reset or advance the errorReported from ExecQueryAndProcessResults
> (plus PQerrorMessage seems to ignore errorReported). I've added an
> additional test to track this behaviour for now as this would probably
> be better discussed in a dedicated thread.
I am not sure if we should change that, actually, as it does not feel
completely wrong to stack these errors. That's a bit confusing,
sure. Perhaps a new libpq API to retrieve stacked errors when we are
in pipeline mode would be more adapted? The design would be
different.
Anyway, I've stared at the result processing code for a couple of
hours, and the branches we're taking for the pipeline modes seem to be
rather right the way you have implemented them. The docs, comments
and tests needed quite a few tweaks and edits to be more consistent.
There were some grammar mistakes, some frenchisms.
I'm hoping that there won't be any issues, but let's be honest, I am
definitely sure there will be some more tuning required. It comes
down to if we want this set of features, and I do to be able to expand
tests in core with the extended query protocol and pipelines, knowing
that there is an ask for out-of-core projects. This one being
reachable with a COPY gave me a huge smile:
+message type 0x5a arrived from server while idle
So let's take one step here, I have applied the main patch. I am
really excited by the possibilities all this stuff offers.
Attached are the remaining pieces, split here because they are
different bullet points:
- Tests for implicit transactions with various commands, with some
edits.
- Prompt support, with more edits.
I'm putting these on standby for a few days, to let the buildfarm
digest the main change.
--
Michael
Attachments:
[text/x-diff] v10-0001-Add-tests-with-implicit-transaction-blocks-and-p.patch (5.8K, ../../[email protected]/2-v10-0001-Add-tests-with-implicit-transaction-blocks-and-p.patch)
download | inline diff:
From 035f19d120b181094b00dee21acb0b4ce4cfc1e1 Mon Sep 17 00:00:00 2001
From: Michael Paquier <[email protected]>
Date: Fri, 21 Feb 2025 11:27:36 +0900
Subject: [PATCH v10 1/2] Add tests with implicit transaction blocks and
pipelines
This checks interactions with $subject for the following commands that
have dedicated behaviors in transaction blocks:
- SET LOCAL
- REINDEX CONCURRENTLY
- VACUUM
- Subtransactions
---
src/test/regress/expected/psql_pipeline.out | 112 ++++++++++++++++++++
src/test/regress/sql/psql_pipeline.sql | 72 +++++++++++++
2 files changed, 184 insertions(+)
diff --git src/test/regress/expected/psql_pipeline.out src/test/regress/expected/psql_pipeline.out
index bdf5a99d09b0..f4603d2b66ae 100644
--- src/test/regress/expected/psql_pipeline.out
+++ src/test/regress/expected/psql_pipeline.out
@@ -585,5 +585,117 @@ PQsendQuery not allowed in pipeline mode
1
(1 row)
+--
+-- Pipelines and transaction blocks
+--
+-- SET LOCAL will issue a warning when modifying a GUC outside of a
+-- transaction block. The change will still be valid as a pipeline
+-- runs within an implicit transaction block. Sending a sync will
+-- commit the implicit transaction block. The first command after a
+-- sync will not be seen as belonging to a pipeline.
+\startpipeline
+SET LOCAL statement_timeout='1h' \bind \g
+SHOW statement_timeout \bind \g
+\syncpipeline
+SHOW statement_timeout \bind \g
+SET LOCAL statement_timeout='2h' \bind \g
+SHOW statement_timeout \bind \g
+\endpipeline
+WARNING: SET LOCAL can only be used in transaction blocks
+ statement_timeout
+-------------------
+ 1h
+(1 row)
+
+ statement_timeout
+-------------------
+ 0
+(1 row)
+
+ statement_timeout
+-------------------
+ 2h
+(1 row)
+
+-- REINDEX CONCURRENTLY fails if not the first command in a pipeline.
+\startpipeline
+SELECT $1 \bind 1 \g
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ERROR: REINDEX CONCURRENTLY cannot run inside a transaction block
+-- REINDEX CONCURRENTLY works if it is the first command in a pipeline.
+\startpipeline
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ ?column?
+----------
+ 2
+(1 row)
+
+-- Subtransactions are not allowed in a pipeline.
+\startpipeline
+SAVEPOINT a \bind \g
+SELECT $1 \bind 1 \g
+ROLLBACK TO SAVEPOINT a \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ERROR: SAVEPOINT can only be used in transaction blocks
+-- LOCK fails as the first command in a pipeline, as not seen in an
+-- implicit transaction block.
+\startpipeline
+LOCK psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ERROR: LOCK TABLE can only be used in transaction blocks
+-- LOCK succeeds as it is not the first command in a pipeline,
+-- seen in an implicit transaction block.
+\startpipeline
+SELECT $1 \bind 1 \g
+LOCK psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+-- VACUUM works as the first command in a pipeline.
+\startpipeline
+VACUUM psql_pipeline \bind \g
+\endpipeline
+-- VACUUM fails when not the first command in a pipeline.
+\startpipeline
+SELECT 1 \bind \g
+VACUUM psql_pipeline \bind \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ERROR: VACUUM cannot run inside a transaction block
+-- VACUUM works after a \syncpipeline.
+\startpipeline
+SELECT 1 \bind \g
+\syncpipeline
+VACUUM psql_pipeline \bind \g
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
-- Clean up
DROP TABLE psql_pipeline;
diff --git src/test/regress/sql/psql_pipeline.sql src/test/regress/sql/psql_pipeline.sql
index 38e48054eeb9..ec62e6c5f247 100644
--- src/test/regress/sql/psql_pipeline.sql
+++ src/test/regress/sql/psql_pipeline.sql
@@ -350,5 +350,77 @@ SELECT 1;
SELECT 1;
\endpipeline
+--
+-- Pipelines and transaction blocks
+--
+
+-- SET LOCAL will issue a warning when modifying a GUC outside of a
+-- transaction block. The change will still be valid as a pipeline
+-- runs within an implicit transaction block. Sending a sync will
+-- commit the implicit transaction block. The first command after a
+-- sync will not be seen as belonging to a pipeline.
+\startpipeline
+SET LOCAL statement_timeout='1h' \bind \g
+SHOW statement_timeout \bind \g
+\syncpipeline
+SHOW statement_timeout \bind \g
+SET LOCAL statement_timeout='2h' \bind \g
+SHOW statement_timeout \bind \g
+\endpipeline
+
+-- REINDEX CONCURRENTLY fails if not the first command in a pipeline.
+\startpipeline
+SELECT $1 \bind 1 \g
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- REINDEX CONCURRENTLY works if it is the first command in a pipeline.
+\startpipeline
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- Subtransactions are not allowed in a pipeline.
+\startpipeline
+SAVEPOINT a \bind \g
+SELECT $1 \bind 1 \g
+ROLLBACK TO SAVEPOINT a \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- LOCK fails as the first command in a pipeline, as not seen in an
+-- implicit transaction block.
+\startpipeline
+LOCK psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- LOCK succeeds as it is not the first command in a pipeline,
+-- seen in an implicit transaction block.
+\startpipeline
+SELECT $1 \bind 1 \g
+LOCK psql_pipeline \bind \g
+SELECT $1 \bind 2 \g
+\endpipeline
+
+-- VACUUM works as the first command in a pipeline.
+\startpipeline
+VACUUM psql_pipeline \bind \g
+\endpipeline
+
+-- VACUUM fails when not the first command in a pipeline.
+\startpipeline
+SELECT 1 \bind \g
+VACUUM psql_pipeline \bind \g
+\endpipeline
+
+-- VACUUM works after a \syncpipeline.
+\startpipeline
+SELECT 1 \bind \g
+\syncpipeline
+VACUUM psql_pipeline \bind \g
+\endpipeline
+
-- Clean up
DROP TABLE psql_pipeline;
--
2.47.2
[text/x-diff] v10-0002-Add-prompt-interpolation-and-variables-for-psql-.patch (6.7K, ../../[email protected]/3-v10-0002-Add-prompt-interpolation-and-variables-for-psql-.patch)
download | inline diff:
From b183638bb31c75fab28ecf67112304b52bb12dce Mon Sep 17 00:00:00 2001
From: Anthonin Bonnefoy <[email protected]>
Date: Tue, 18 Feb 2025 15:32:31 +0100
Subject: [PATCH v10 2/2] Add prompt interpolation and variables for psql
pipeline
Add %P prompt interpolation that reports the status pipeline: on, off or
abort. Additionally, 3 new special variables are added to report the
status of an ongoing pipeline:
- PIPELINE_SYNC_COUNT: reports the number of piped sync
- PIPELINE_COMMAND_COUNT: reports the number of piped commands, a
command being either \bind, \bind_named, \close or \parse
- PIPELINE_RESULT_COUNT: reports the results available to read with
\getresults
---
src/bin/psql/common.c | 27 +++++++++++++++++-
src/bin/psql/prompt.c | 14 ++++++++++
src/bin/psql/startup.c | 5 ++++
doc/src/sgml/ref/psql-ref.sgml | 50 ++++++++++++++++++++++++++++++++++
4 files changed, 95 insertions(+), 1 deletion(-)
diff --git src/bin/psql/common.c src/bin/psql/common.c
index bc8c40898f71..ed340a466f99 100644
--- src/bin/psql/common.c
+++ src/bin/psql/common.c
@@ -524,6 +524,26 @@ SetShellResultVariables(int wait_result)
}
+/*
+ * Set special pipeline variables
+ * - PIPELINE_SYNC_COUNT: The number of piped syncs
+ * - PIPELINE_COMMAND_COUNT: The number of piped commands
+ * - PIPELINE_RESULT_COUNT: The number of results available to read
+ */
+static void
+SetPipelineVariables(void)
+{
+ char buf[32];
+
+ snprintf(buf, sizeof(buf), "%d", pset.piped_syncs);
+ SetVariable(pset.vars, "PIPELINE_SYNC_COUNT", buf);
+ snprintf(buf, sizeof(buf), "%d", pset.piped_commands);
+ SetVariable(pset.vars, "PIPELINE_COMMAND_COUNT", buf);
+ snprintf(buf, sizeof(buf), "%d", pset.available_results);
+ SetVariable(pset.vars, "PIPELINE_RESULT_COUNT", buf);
+}
+
+
/*
* ClearOrSaveResult
*
@@ -1661,6 +1681,8 @@ ExecQueryAndProcessResults(const char *query,
CheckConnection();
+ SetPipelineVariables();
+
return -1;
}
@@ -1669,8 +1691,10 @@ ExecQueryAndProcessResults(const char *query,
{
/*
* We are in a pipeline and have not reached the pipeline end, or
- * there was no request to read pipeline results, exit.
+ * there was no request to read pipeline results. Update the psql
+ * variables tracking the pipeline activity and exit.
*/
+ SetPipelineVariables();
return 1;
}
@@ -2105,6 +2129,7 @@ ExecQueryAndProcessResults(const char *query,
Assert(pset.available_results == 0);
}
Assert(pset.requested_results == 0);
+ SetPipelineVariables();
/* may need this to recover from conn loss during COPY */
if (!CheckConnection())
diff --git src/bin/psql/prompt.c src/bin/psql/prompt.c
index 08a14feb3c3f..3aa7d2d06c80 100644
--- src/bin/psql/prompt.c
+++ src/bin/psql/prompt.c
@@ -31,6 +31,7 @@
* sockets, "[local:/dir/name]" if not default
* %m - like %M, but hostname only (before first dot), or always "[local]"
* %p - backend pid
+ * %P - pipeline status: on, off or abort
* %> - database server port number
* %n - database user name
* %s - service
@@ -181,6 +182,19 @@ get_prompt(promptStatus_t status, ConditionalStack cstack)
snprintf(buf, sizeof(buf), "%d", pid);
}
break;
+ /* pipeline status */
+ case 'P':
+ {
+ PGpipelineStatus status = PQpipelineStatus(pset.db);
+
+ if (status == PQ_PIPELINE_ON)
+ strlcpy(buf, "on", sizeof(buf));
+ else if (status == PQ_PIPELINE_ABORTED)
+ strlcpy(buf, "abort", sizeof(buf));
+ else
+ strlcpy(buf, "off", sizeof(buf));
+ break;
+ }
case '0':
case '1':
diff --git src/bin/psql/startup.c src/bin/psql/startup.c
index 703f3f582c17..5018eedf1e57 100644
--- src/bin/psql/startup.c
+++ src/bin/psql/startup.c
@@ -205,6 +205,11 @@ main(int argc, char *argv[])
SetVariable(pset.vars, "PROMPT3", DEFAULT_PROMPT3);
SetVariableBool(pset.vars, "SHOW_ALL_RESULTS");
+ /* Initialize pipeline variables */
+ SetVariable(pset.vars, "PIPELINE_SYNC_COUNT", "0");
+ SetVariable(pset.vars, "PIPELINE_COMMAND_COUNT", "0");
+ SetVariable(pset.vars, "PIPELINE_RESULT_COUNT", "0");
+
parse_psql_options(argc, argv, &options);
/*
diff --git doc/src/sgml/ref/psql-ref.sgml doc/src/sgml/ref/psql-ref.sgml
index a8a0b694bbf0..32130242e9c8 100644
--- doc/src/sgml/ref/psql-ref.sgml
+++ doc/src/sgml/ref/psql-ref.sgml
@@ -3727,6 +3727,12 @@ testdb=> <userinput>\setenv LESS -imx4F</userinput>
generate one result to get.
</para>
+ <para>
+ When pipeline mode is active, a dedicated prompt variable is available
+ to report the pipeline status.
+ See <xref linkend="app-psql-prompting-p-uc"/> for more details
+ </para>
+
<para>
Example:
<programlisting>
@@ -4501,6 +4507,39 @@ bar
</listitem>
</varlistentry>
+ <varlistentry id="app-psql-variables-pipeline-command-count">
+ <term><varname>PIPELINE_COMMAND_COUNT</varname></term>
+ <listitem>
+ <para>
+ The number of commands generated by <literal>\bind</literal>,
+ <literal>\bind_named</literal>, <literal>\close</literal> or
+ <literal>\parse</literal> queued in an ongoing pipeline.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry id="app-psql-variables-pipeline-result-count">
+ <term><varname>PIPELINE_RESULT_COUNT</varname></term>
+ <listitem>
+ <para>
+ The number of commands of an ongoing pipeline that were followed
+ by either a <command>\flushrequest</command> or a
+ <command>\syncpipeline</command>, forcing the server to send the
+ results. These results can be retrieved with
+ <command>\getresults</command>.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry id="app-psql-variables-pipeline-sync-count">
+ <term><varname>PIPELINE_SYNC_COUNT</varname></term>
+ <listitem>
+ <para>
+ The number of sync messages queued in an ongoing pipeline.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry id="app-psql-variables-port">
<term><varname>PORT</varname></term>
<listitem>
@@ -4900,6 +4939,17 @@ testdb=> <userinput>INSERT INTO my_table VALUES (:'content');</userinput>
</listitem>
</varlistentry>
+ <varlistentry id="app-psql-prompting-p-uc">
+ <term><literal>%P</literal></term>
+ <listitem>
+ <para>
+ Pipeline status: <literal>off</literal> when not in a pipeline,
+ <literal>on</literal> when in an ongoing pipeline or
+ <literal>abort</literal> when in an aborted pipeline.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry id="app-psql-prompting-r">
<term><literal>%R</literal></term>
<listitem>
--
2.47.2
[application/pgp-signature] signature.asc (833B, ../../[email protected]/4-signature.asc)
download
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-02-25 01:11 Michael Paquier <[email protected]>
parent: Michael Paquier <[email protected]>
0 siblings, 2 replies; 41+ messages in thread
From: Michael Paquier @ 2025-02-25 01:11 UTC (permalink / raw)
To: Anthonin Bonnefoy <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; pgsql-hackers
On Fri, Feb 21, 2025 at 11:33:41AM +0900, Michael Paquier wrote:
> Attached are the remaining pieces, split here because they are
> different bullet points:
> - Tests for implicit transactions with various commands, with some
> edits.
> - Prompt support, with more edits.
>
> I'm putting these on standby for a few days, to let the buildfarm
> digest the main change.
Initial digestion has gone well. The remaining pieces have been done
as 3ce357584e79 and a4e986ef5a46. For the prompt part, I have added a
couple of tests with \echo and the variables. The patch felt
incomplete without these. Perhaps we could extend them more, at least
we have a start point.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-02-25 07:30 Anthonin Bonnefoy <[email protected]>
parent: Michael Paquier <[email protected]>
1 sibling, 0 replies; 41+ messages in thread
From: Anthonin Bonnefoy @ 2025-02-25 07:30 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; pgsql-hackers
On Tue, Feb 25, 2025 at 2:11 AM Michael Paquier <[email protected]> wrote:
> For the prompt part, I have added a
> couple of tests with \echo and the variables. The patch felt
> incomplete without these. Perhaps we could extend them more, at least
> we have a start point.
Good catch. I guess that's also another benefit of having special
variables, as it makes it easier to check the pipeline state.
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-02-28 16:31 Daniel Verite <[email protected]>
parent: Anthonin Bonnefoy <[email protected]>
1 sibling, 1 reply; 41+ messages in thread
From: Daniel Verite @ 2025-02-28 16:31 UTC (permalink / raw)
To: Anthonin Bonnefoy <[email protected]>; +Cc: Michael Paquier <[email protected]>; Jelte Fennema-Nio <[email protected]>; pgsql-hackers
Anthonin Bonnefoy wrote:
> > What is the reasoning here behind this restriction? \gx is a wrapper
> > of \g with expanded mode on, but it is also possible to call \g with
> > expanded=on, bypassing this restriction.
>
> The issue is that \gx enables expanded mode for the duration of the
> query and immediately reset it in sendquery_cleanup. With pipelining,
> the command is piped and displaying is done by either \endpipeline or
> \getresults, so the flag change has no impact. Forbidding it was a way
> to make it clearer that it won't have the expected effect
But it's not just \gx
The following invocations don't respect the desired output destination
and formats (ignoring them), when a pipeline is active:
select ... \bind \g filename
select ... \bind \g |program
select ... \bind \g (format=unaligned tuples_only=on)
Just like for \gx the problem is that in a pipeline, sending the query
is not followed by getting the results, and the output properties
of a query are lost in between.
Best regards,
--
Daniel Vérité
https://postgresql.verite.pro/
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-04 00:56 Michael Paquier <[email protected]>
parent: Daniel Verite <[email protected]>
0 siblings, 1 reply; 41+ messages in thread
From: Michael Paquier @ 2025-03-04 00:56 UTC (permalink / raw)
To: Daniel Verite <[email protected]>; +Cc: Anthonin Bonnefoy <[email protected]>; Jelte Fennema-Nio <[email protected]>; pgsql-hackers
(My previous message did not reach the lists, so re-sending with some
edits.)
On Fri, Feb 28, 2025 at 05:31:00PM +0100, Daniel Verite wrote:
> The following invocations don't respect the desired output destination
> and formats (ignoring them), when a pipeline is active:
>
> select ... \bind \g filename
> select ... \bind \g |program
> select ... \bind \g (format=unaligned tuples_only=on)
>
> Just like for \gx the problem is that in a pipeline, sending the query
> is not followed by getting the results, and the output properties
> of a query are lost in between.
Right. I completely forgot that these options could be applied with a
simple \g. With the results being decoupled from the execution, one
can argue that the options defined at the moment when the query is
sent and executed should be the moment commanding how the result are
shaped when retrieving a batch with \getresult during a pipeline.
However, this means that we would need to save the set of options from
printQueryOpt when running the query depending on the number of
results we expect, and reapply them in order of the results
expected. We have the APIs to do that, with savePsetInfo() and
restorePsetInfo().
Anyway, can we really say that the set of printQueryOpt saved at
execution is the correct set to use? It is possible to have the
opposite argument and say that we should just apply the printQueryOpt
at the moment where \getresult is run. A benefit of that it to keep
the loop retrieving results simpler in ExecQueryAndProcessResults(),
because we pass down to this call *one* printQueryOpt as "opt".
There's of course the simplicity argument that I do like a lot here,
but applying the printing options at the time of \getresult feels also
more natural to me.
FWIW, I agree that HEAD is unbalanced with its handling of \gx, so we
could do one of the following two things:
1) Ignore any formatting options given to \g, but also allow \gx to
run, documenting that during a pipeline the formatting options are
ignored, and that the set of options defined when doing a \getresult
is the only thing that matters.
2) Invent a new meta-command (suggested by Daniel Verite off-list, and
not published to the lists because I don't know how to do a
reply-all), like a \send, a \push, forbiding entirely \gx and \g when
in a pipeline. If we were to integrate options into this new
meta-command, a split with \g would make an integration easier to
think about. One name suggestion I can come up is \sendpipeline.
I'd consider option 2, based on Daniel's concerns. One line I'm going
to draw here is that we should not go down to manipulations of
printQueryOpt while retrieving batch of results depending on the style
of output that was defined when sending a query in a pipeline.
Anthonin, as the primary author, any thoughts?
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-04 09:31 Anthonin Bonnefoy <[email protected]>
parent: Michael Paquier <[email protected]>
0 siblings, 1 reply; 41+ messages in thread
From: Anthonin Bonnefoy @ 2025-03-04 09:31 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Daniel Verite <[email protected]>; Jelte Fennema-Nio <[email protected]>; pgsql-hackers
On Tue, Mar 4, 2025 at 1:57 AM Michael Paquier <[email protected]> wrote:
> Anyway, can we really say that the set of printQueryOpt saved at
> execution is the correct set to use? It is possible to have the
> opposite argument and say that we should just apply the printQueryOpt
> at the moment where \getresult is run. A benefit of that it to keep
> the loop retrieving results simpler in ExecQueryAndProcessResults(),
> because we pass down to this call *one* printQueryOpt as "opt".
>
> There's of course the simplicity argument that I do like a lot here,
> but applying the printing options at the time of \getresult feels also
> more natural to me.
Saving the printQueryOpt when a command is pushed was an option I had
in mind if that was straightforward to implement. However, even with
savePsetInfo, you will need to save values like gfname and gset_prefix
since it impacts the output (it may make sense to move those in
printQueryOpt). This would also need to be saved for all commands,
like \close or \parse since we don't distinguish if a piped command
generates an output or not. So that definitely looks like it would add
a lot of complexity for limited benefit.
> FWIW, I agree that HEAD is unbalanced with its handling of \gx, so we
> could do one of the following two things:
> 1) Ignore any formatting options given to \g, but also allow \gx to
> run, documenting that during a pipeline the formatting options are
> ignored, and that the set of options defined when doing a \getresult
> is the only thing that matters.
> 2) Invent a new meta-command (suggested by Daniel Verite off-list, and
> not published to the lists because I don't know how to do a
> reply-all), like a \send, a \push, forbiding entirely \gx and \g when
> in a pipeline. If we were to integrate options into this new
> meta-command, a split with \g would make an integration easier to
> think about. One name suggestion I can come up is \sendpipeline.
>
> I'd consider option 2, based on Daniel's concerns. One line I'm going
> to draw here is that we should not go down to manipulations of
> printQueryOpt while retrieving batch of results depending on the style
> of output that was defined when sending a query in a pipeline.
Another possible option would be to directly send the command without
requiring an additional meta-command, like "SELECT 1 \bind". However,
this would make it more painful to introduce new parameters, plus it
makes the \bind and \bind_named inconsistent as it is normally
required to send the result with a separate meta-command.
I like the \sendpipeline option. It makes it clearer that formatting
options are not applicable within a pipeline (at least, in the current
implementation) and I think it would make more sense to have those
formatting options in \getresults or \endpipeline.
I took a stab at creating the \sendpipeline meta-command. I've also
realised there's a small leak where fname is currently not freed on
queries like 'select ... \bind \gx filename' when within a pipeline,
which is fixed in patch 0001.
Attachments:
[application/octet-stream] v01-0001-Fix-fname-leak-on-gx-within-a-pipeline.patch (924B, ../../CAO6_XqqFVQjLjZQiL7xdwLpzZEy1ghO_JWvCFPM_OmwF9s7XdA@mail.gmail.com/2-v01-0001-Fix-fname-leak-on-gx-within-a-pipeline.patch)
download | inline diff:
From 8286539d24698b13f283a9faef6f2aec3093e75a Mon Sep 17 00:00:00 2001
From: Anthonin Bonnefoy <[email protected]>
Date: Tue, 4 Mar 2025 09:04:31 +0100
Subject: Fix fname leak on \gx within a pipeline
While inside a pipeline, \gx is forbidden and will make exec_command_g
exit early. If a filename parameter was provided, the allocated fname
currently isn't freed.
This patch adds the missing free when this happens.
---
src/bin/psql/command.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c
index 0f27bf7a91f..fb0b27568c5 100644
--- a/src/bin/psql/command.c
+++ b/src/bin/psql/command.c
@@ -1739,6 +1739,7 @@ exec_command_g(PsqlScanState scan_state, bool active_branch, const char *cmd)
{
pg_log_error("\\gx not allowed in pipeline mode");
clean_extended_state();
+ free(fname);
return PSQL_CMD_ERROR;
}
--
2.39.5 (Apple Git-154)
[application/octet-stream] v01-0002-psql-Create-new-sendpipeline-meta-command.patch (38.8K, ../../CAO6_XqqFVQjLjZQiL7xdwLpzZEy1ghO_JWvCFPM_OmwF9s7XdA@mail.gmail.com/3-v01-0002-psql-Create-new-sendpipeline-meta-command.patch)
download | inline diff:
From d1773a1075d9aa5f0c870ae6d2ba1c05b6f92e65 Mon Sep 17 00:00:00 2001
From: Anthonin Bonnefoy <[email protected]>
Date: Tue, 4 Mar 2025 09:47:45 +0100
Subject: psql: Create new \sendpipeline meta-command
In the initial pipelining support for psql, \g was reused as a way to
push extended query into an ongoing pipeline. However, all related
meta-command options are not applicable in pipeline mode. Pipeline
output only happens during \getresults or \endpipeline and any change to
output options would have been reset.
To avoid possible confusion, a new \sendpipeline meta-command is
introduced, replacing \g when within a pipeline.
---
doc/src/sgml/ref/psql-ref.sgml | 11 +-
src/bin/psql/command.c | 45 +++-
src/bin/psql/help.c | 1 +
src/bin/psql/tab-complete.in.c | 2 +-
src/test/regress/expected/psql.out | 1 +
src/test/regress/expected/psql_pipeline.out | 245 +++++++++++---------
src/test/regress/sql/psql.sql | 1 +
src/test/regress/sql/psql_pipeline.sql | 230 +++++++++---------
8 files changed, 310 insertions(+), 226 deletions(-)
diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml
index cedccc14129..cddf6e07531 100644
--- a/doc/src/sgml/ref/psql-ref.sgml
+++ b/doc/src/sgml/ref/psql-ref.sgml
@@ -3677,6 +3677,7 @@ testdb=> <userinput>\setenv LESS -imx4F</userinput>
<varlistentry id="app-psql-meta-command-pipeline">
<term><literal>\startpipeline</literal></term>
+ <term><literal>\sendpipeline</literal></term>
<term><literal>\syncpipeline</literal></term>
<term><literal>\endpipeline</literal></term>
<term><literal>\flushrequest</literal></term>
@@ -3701,10 +3702,10 @@ testdb=> <userinput>\setenv LESS -imx4F</userinput>
queries need to be sent using the meta-commands
<literal>\bind</literal>, <literal>\bind_named</literal>,
<literal>\close</literal> or <literal>\parse</literal>. While a
- pipeline is ongoing, <literal>\g</literal> will append the current
- query buffer to the pipeline. Other meta-commands like
- <literal>\gx</literal> or <literal>\gdesc</literal> are not allowed
- in pipeline mode.
+ pipeline is ongoing, <literal>\sendpipeline</literal> will append the
+ current query buffer to the pipeline. Other meta-commands like
+ <literal>\g</literal>, <literal>\gx</literal> or <literal>\gdesc</literal>
+ are not allowed in pipeline mode.
</para>
<para>
@@ -3738,7 +3739,7 @@ testdb=> <userinput>\setenv LESS -imx4F</userinput>
Example:
<programlisting>
\startpipeline
-SELECT 1 \bind \g
+SELECT 1 \bind \sendpipeline
\flushrequest
\getresults
\endpipeline
diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c
index fb0b27568c5..7670c20b29e 100644
--- a/src/bin/psql/command.c
+++ b/src/bin/psql/command.c
@@ -137,6 +137,7 @@ static backslashResult exec_command_setenv(PsqlScanState scan_state, bool active
static backslashResult exec_command_sf_sv(PsqlScanState scan_state, bool active_branch,
const char *cmd, bool is_func);
static backslashResult exec_command_startpipeline(PsqlScanState scan_state, bool active_branch);
+static backslashResult exec_command_sendpipeline(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_syncpipeline(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_t(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_T(PsqlScanState scan_state, bool active_branch);
@@ -427,6 +428,8 @@ exec_command(const char *cmd,
status = exec_command_sf_sv(scan_state, active_branch, cmd, false);
else if (strcmp(cmd, "startpipeline") == 0)
status = exec_command_startpipeline(scan_state, active_branch);
+ else if (strcmp(cmd, "sendpipeline") == 0)
+ status = exec_command_sendpipeline(scan_state, active_branch);
else if (strcmp(cmd, "syncpipeline") == 0)
status = exec_command_syncpipeline(scan_state, active_branch);
else if (strcmp(cmd, "t") == 0)
@@ -1734,10 +1737,9 @@ exec_command_g(PsqlScanState scan_state, bool active_branch, const char *cmd)
if (status == PSQL_CMD_SKIP_LINE && active_branch)
{
- if (strcmp(cmd, "gx") == 0 &&
- PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
{
- pg_log_error("\\gx not allowed in pipeline mode");
+ pg_log_error("\\%s not allowed in pipeline mode", cmd);
clean_extended_state();
free(fname);
return PSQL_CMD_ERROR;
@@ -2979,6 +2981,43 @@ exec_command_startpipeline(PsqlScanState scan_state, bool active_branch)
return status;
}
+/*
+ * \sendpipeline -- send an extended query to an ongoing pipeline
+ */
+static backslashResult
+exec_command_sendpipeline(PsqlScanState scan_state, bool active_branch)
+{
+ backslashResult status = PSQL_CMD_SKIP_LINE;
+
+ if (active_branch)
+ {
+ if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ if (pset.send_mode == PSQL_SEND_EXTENDED_QUERY_PREPARED ||
+ pset.send_mode == PSQL_SEND_EXTENDED_QUERY_PARAMS)
+ {
+ status = PSQL_CMD_SEND;
+ }
+ else
+ {
+ pg_log_error("\\sendpipeline must be used after \\bind or \\bind_named");
+ clean_extended_state();
+ return PSQL_CMD_ERROR;
+ }
+ }
+ else
+ {
+ pg_log_error("\\sendpipeline not allowed outside of pipeline mode");
+ clean_extended_state();
+ return PSQL_CMD_ERROR;
+ }
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return status;
+}
+
/*
* \syncpipeline -- send a sync message to an active pipeline
*/
diff --git a/src/bin/psql/help.c b/src/bin/psql/help.c
index 714b8619233..edfc794b76f 100644
--- a/src/bin/psql/help.c
+++ b/src/bin/psql/help.c
@@ -182,6 +182,7 @@ slashUsage(unsigned short int pager)
HELP0(" \\parse STMT_NAME create a prepared statement\n");
HELP0(" \\q quit psql\n");
HELP0(" \\startpipeline enter pipeline mode\n");
+ HELP0(" \\sendpipeline send an extended query to an ongoing pipeline\n");
HELP0(" \\syncpipeline add a synchronisation point to an ongoing pipeline\n");
HELP0(" \\watch [[i=]SEC] [c=N] [m=MIN]\n"
" execute query every SEC seconds, up to N times,\n"
diff --git a/src/bin/psql/tab-complete.in.c b/src/bin/psql/tab-complete.in.c
index 8432be641ac..b2de63a5b74 100644
--- a/src/bin/psql/tab-complete.in.c
+++ b/src/bin/psql/tab-complete.in.c
@@ -1895,7 +1895,7 @@ psql_completion(const char *text, int start, int end)
"\\parse", "\\password", "\\print", "\\prompt", "\\pset",
"\\qecho", "\\quit",
"\\reset",
- "\\s", "\\set", "\\setenv", "\\sf", "\\startpipeline", "\\sv", "\\syncpipeline",
+ "\\s", "\\set", "\\setenv", "\\sf", "\\startpipeline", "\\sendpipeline", "\\sv", "\\syncpipeline",
"\\t", "\\T", "\\timing",
"\\unset",
"\\x",
diff --git a/src/test/regress/expected/psql.out b/src/test/regress/expected/psql.out
index 6543e90de75..ee6f8671bea 100644
--- a/src/test/regress/expected/psql.out
+++ b/src/test/regress/expected/psql.out
@@ -4711,6 +4711,7 @@ invalid command \lo
\sf whole_line
\sv whole_line
\startpipeline
+ \sendpipeline
\syncpipeline
\t arg1
\T arg1
diff --git a/src/test/regress/expected/psql_pipeline.out b/src/test/regress/expected/psql_pipeline.out
index 3df2415a840..53f2edfd986 100644
--- a/src/test/regress/expected/psql_pipeline.out
+++ b/src/test/regress/expected/psql_pipeline.out
@@ -4,7 +4,7 @@
CREATE TABLE psql_pipeline(a INTEGER PRIMARY KEY, s TEXT);
-- Single query
\startpipeline
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
?column?
----------
@@ -13,9 +13,9 @@ SELECT $1 \bind 'val1' \g
-- Multiple queries
\startpipeline
-SELECT $1 \bind 'val1' \g
-SELECT $1, $2 \bind 'val2' 'val3' \g
-SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1 \bind 'val1' \sendpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
\endpipeline
?column?
----------
@@ -35,10 +35,10 @@ SELECT $1, $2 \bind 'val2' 'val3' \g
-- Test \flush
\startpipeline
\flush
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\flush
-SELECT $1, $2 \bind 'val2' 'val3' \g
-SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
\endpipeline
?column?
----------
@@ -63,12 +63,12 @@ SELECT $1, $2 \bind 'val2' 'val3' \g
0
\echo :PIPELINE_RESULT_COUNT
0
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\syncpipeline
\syncpipeline
-SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
\syncpipeline
-SELECT $1, $2 \bind 'val4' 'val5' \g
+SELECT $1, $2 \bind 'val4' 'val5' \sendpipeline
\echo :PIPELINE_COMMAND_COUNT
1
\echo :PIPELINE_SYNC_COUNT
@@ -94,7 +94,7 @@ SELECT $1, $2 \bind 'val4' 'val5' \g
-- \startpipeline should not have any effect if already in a pipeline.
\startpipeline
\startpipeline
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
?column?
----------
@@ -103,24 +103,24 @@ SELECT $1 \bind 'val1' \g
-- Convert an implicit transaction block to an explicit transaction block.
\startpipeline
-INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
-BEGIN \bind \g
-INSERT INTO psql_pipeline VALUES ($1) \bind 2 \g
-ROLLBACK \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \sendpipeline
+BEGIN \bind \sendpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 2 \sendpipeline
+ROLLBACK \bind \sendpipeline
\endpipeline
-- Multiple explicit transactions
\startpipeline
-BEGIN \bind \g
-INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
-ROLLBACK \bind \g
-BEGIN \bind \g
-INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
-COMMIT \bind \g
+BEGIN \bind \sendpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \sendpipeline
+ROLLBACK \bind \sendpipeline
+BEGIN \bind \sendpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \sendpipeline
+COMMIT \bind \sendpipeline
\endpipeline
-- COPY FROM STDIN
\startpipeline
-SELECT $1 \bind 'val1' \g
-COPY psql_pipeline FROM STDIN \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+COPY psql_pipeline FROM STDIN \bind \sendpipeline
\endpipeline
?column?
----------
@@ -129,8 +129,8 @@ COPY psql_pipeline FROM STDIN \bind \g
-- COPY FROM STDIN with \flushrequest + \getresults
\startpipeline
-SELECT $1 \bind 'val1' \g
-COPY psql_pipeline FROM STDIN \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+COPY psql_pipeline FROM STDIN \bind \sendpipeline
\flushrequest
\getresults
?column?
@@ -142,8 +142,8 @@ message type 0x5a arrived from server while idle
\endpipeline
-- COPY FROM STDIN with \syncpipeline + \getresults
\startpipeline
-SELECT $1 \bind 'val1' \g
-COPY psql_pipeline FROM STDIN \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+COPY psql_pipeline FROM STDIN \bind \sendpipeline
\syncpipeline
\getresults
?column?
@@ -154,8 +154,8 @@ COPY psql_pipeline FROM STDIN \bind \g
\endpipeline
-- COPY TO STDOUT
\startpipeline
-SELECT $1 \bind 'val1' \g
-copy psql_pipeline TO STDOUT \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+copy psql_pipeline TO STDOUT \bind \sendpipeline
\endpipeline
?column?
----------
@@ -168,8 +168,8 @@ copy psql_pipeline TO STDOUT \bind \g
4 test4
-- COPY TO STDOUT with \flushrequest + \getresults
\startpipeline
-SELECT $1 \bind 'val1' \g
-copy psql_pipeline TO STDOUT \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+copy psql_pipeline TO STDOUT \bind \sendpipeline
\flushrequest
\getresults
?column?
@@ -184,8 +184,8 @@ copy psql_pipeline TO STDOUT \bind \g
\endpipeline
-- COPY TO STDOUT with \syncpipeline + \getresults
\startpipeline
-SELECT $1 \bind 'val1' \g
-copy psql_pipeline TO STDOUT \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+copy psql_pipeline TO STDOUT \bind \sendpipeline
\syncpipeline
\getresults
?column?
@@ -203,14 +203,14 @@ copy psql_pipeline TO STDOUT \bind \g
SELECT $1 \parse ''
SELECT $1, $2 \parse ''
SELECT $2 \parse pipeline_1
-\bind_named '' 1 2 \g
-\bind_named pipeline_1 2 \g
+\bind_named '' 1 2 \sendpipeline
+\bind_named pipeline_1 2 \sendpipeline
\endpipeline
ERROR: could not determine data type of parameter $1
-- \getresults displays all results preceding a \flushrequest.
\startpipeline
-SELECT $1 \bind 1 \g
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 1 \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\flushrequest
\getresults
?column?
@@ -226,8 +226,8 @@ SELECT $1 \bind 2 \g
\endpipeline
-- \getresults displays all results preceding a \syncpipeline.
\startpipeline
-SELECT $1 \bind 1 \g
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 1 \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\syncpipeline
\getresults
?column?
@@ -245,7 +245,7 @@ SELECT $1 \bind 2 \g
\startpipeline
\getresults
No pending results to get
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\getresults
No pending results to get
\flushrequest
@@ -259,9 +259,9 @@ No pending results to get
No pending results to get
-- \getresults only fetches results preceding a \flushrequest.
\startpipeline
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\flushrequest
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\getresults
?column?
----------
@@ -276,9 +276,9 @@ SELECT $1 \bind 2 \g
-- \getresults only fetches results preceding a \syncpipeline.
\startpipeline
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\syncpipeline
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\getresults
?column?
----------
@@ -294,7 +294,7 @@ SELECT $1 \bind 2 \g
-- Use pipeline with chunked results for both \getresults and \endpipeline.
\startpipeline
\set FETCH_COUNT 10
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\flushrequest
\getresults
?column?
@@ -302,7 +302,7 @@ SELECT $1 \bind 2 \g
2
(1 row)
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
?column?
----------
@@ -312,9 +312,9 @@ SELECT $1 \bind 2 \g
\unset FETCH_COUNT
-- \getresults with specific number of requested results.
\startpipeline
-SELECT $1 \bind 1 \g
-SELECT $1 \bind 2 \g
-SELECT $1 \bind 3 \g
+SELECT $1 \bind 1 \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
+SELECT $1 \bind 3 \sendpipeline
\echo :PIPELINE_SYNC_COUNT
0
\syncpipeline
@@ -330,7 +330,7 @@ SELECT $1 \bind 3 \g
\echo :PIPELINE_RESULT_COUNT
2
-SELECT $1 \bind 4 \g
+SELECT $1 \bind 4 \sendpipeline
\getresults 3
?column?
----------
@@ -354,7 +354,7 @@ SELECT $1 \bind 4 \g
\startpipeline
\syncpipeline
\syncpipeline
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
\flushrequest
\getresults 2
\getresults 1
@@ -366,9 +366,9 @@ SELECT $1 \bind 1 \g
\endpipeline
-- \getresults 0 should get all the results.
\startpipeline
-SELECT $1 \bind 1 \g
-SELECT $1 \bind 2 \g
-SELECT $1 \bind 3 \g
+SELECT $1 \bind 1 \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
+SELECT $1 \bind 3 \sendpipeline
\syncpipeline
\getresults 0
?column?
@@ -398,7 +398,7 @@ cannot send pipeline when not in pipeline mode
\startpipeline
SELECT 1;
PQsendQuery not allowed in pipeline mode
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
?column?
----------
@@ -408,9 +408,9 @@ SELECT $1 \bind 'val1' \g
-- After an aborted pipeline, commands after a \syncpipeline should be
-- displayed.
\startpipeline
-SELECT $1 \bind \g
+SELECT $1 \bind \sendpipeline
\syncpipeline
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
\endpipeline
ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
?column?
@@ -421,23 +421,23 @@ ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
-- For an incorrect number of parameters, the pipeline is aborted and
-- the following queries will not be executed.
\startpipeline
-SELECT \bind 'val1' \g
-SELECT $1 \bind 'val1' \g
+SELECT \bind 'val1' \sendpipeline
+SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
ERROR: bind message supplies 1 parameters, but prepared statement "" requires 0
-- An explicit transaction with an error needs to be rollbacked after
-- the pipeline.
\startpipeline
-BEGIN \bind \g
-INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
-ROLLBACK \bind \g
+BEGIN \bind \sendpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \sendpipeline
+ROLLBACK \bind \sendpipeline
\endpipeline
ERROR: duplicate key value violates unique constraint "psql_pipeline_pkey"
DETAIL: Key (a)=(1) already exists.
ROLLBACK;
-- \watch sends a simple query, something not allowed within a pipeline.
\startpipeline
-SELECT \bind \g
+SELECT \bind \sendpipeline
\watch 1
PQsendQuery not allowed in pipeline mode
@@ -450,7 +450,7 @@ PQsendQuery not allowed in pipeline mode
\startpipeline
SELECT $1 \bind 1 \gdesc
synchronous command execution functions are not allowed in pipeline mode
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
\endpipeline
?column?
----------
@@ -465,19 +465,33 @@ SELECT $1 as k, $2 as l \parse 'second'
\gset not allowed in pipeline mode
\bind_named second 1 2 \gset pref02_ \echo :pref02_i :pref02_j
\gset not allowed in pipeline mode
-\bind_named '' 1 2 \g
+\bind_named '' 1 2 \sendpipeline
\endpipeline
i | j
---+---
1 | 2
(1 row)
--- \gx is not allowed, pipeline should still be usable.
+-- \g and \gx are not allowed, pipeline should still be usable.
\startpipeline
+SELECT $1 \bind 1 \g
+\g not allowed in pipeline mode
+SELECT $1 \bind 1 \g filename
+\g not allowed in pipeline mode
+SELECT $1 \bind 1 \g |cat
+\g not allowed in pipeline mode
+SELECT $1 \bind 1 \g (format=unaligned tuples_only=on)
+\g not allowed in pipeline mode
SELECT $1 \bind 1 \gx
\gx not allowed in pipeline mode
+SELECT $1 \bind 1 \gx filename
+\gx not allowed in pipeline mode
+SELECT $1 \bind 1 \gx |cat
+\gx not allowed in pipeline mode
+SELECT $1 \bind 1 \gx (format=unaligned tuples_only=on)
+\gx not allowed in pipeline mode
\reset
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
\endpipeline
?column?
----------
@@ -487,54 +501,63 @@ SELECT $1 \bind 1 \g
-- \gx warning should be emitted in an aborted pipeline, with
-- pipeline still usable.
\startpipeline
-SELECT $1 \bind \g
+SELECT $1 \bind \sendpipeline
\flushrequest
\getresults
ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
SELECT $1 \bind 1 \gx
\gx not allowed in pipeline mode
\endpipeline
+-- \sendpipeline is not allowed outside of a pipeline
+\sendpipeline
+\sendpipeline not allowed outside of pipeline mode
+SELECT $1 \bind 1 \sendpipeline
+\sendpipeline not allowed outside of pipeline mode
+\reset
+-- \sendpipeline is not allowed if not preceded by \bind or \bind_named
+\startpipeline
+\sendpipeline
+\sendpipeline must be used after \bind or \bind_named
+SELECT 1 \sendpipeline
+\sendpipeline must be used after \bind or \bind_named
+\endpipeline
-- \gexec is not allowed, pipeline should still be usable.
\startpipeline
SELECT 'INSERT INTO psql_pipeline(a) SELECT generate_series(1, 10)' \parse 'insert_stmt'
\bind_named insert_stmt \gexec
\gexec not allowed in pipeline mode
-\bind_named insert_stmt \g
+\bind_named insert_stmt \sendpipeline
SELECT COUNT(*) FROM psql_pipeline \bind \g
+\g not allowed in pipeline mode
\endpipeline
?column?
------------------------------------------------------------
INSERT INTO psql_pipeline(a) SELECT generate_series(1, 10)
(1 row)
- count
--------
- 4
-(1 row)
-
-- After an error, pipeline is aborted and requires \syncpipeline to be
-- reusable.
\startpipeline
-SELECT $1 \bind \g
-SELECT $1 \bind 1 \g
+SELECT $1 \bind \sendpipeline
+SELECT $1 \bind 1 \sendpipeline
SELECT $1 \parse a
-\bind_named a 1 \g
+\bind_named a 1 \sendpipeline
\close a
\flushrequest
\getresults
ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
-- Pipeline is aborted.
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
SELECT $1 \parse a
-\bind_named a 1 \g
+\bind_named a 1 \sendpipeline
\close a
-- Sync allows pipeline to recover.
\syncpipeline
\getresults
Pipeline aborted, command did not run
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
SELECT $1 \parse a
-\bind_named a 1 \g
+\bind_named a 1 \sendpipeline
\close a
\flushrequest
\getresults
@@ -551,10 +574,10 @@ SELECT $1 \parse a
\endpipeline
-- In an aborted pipeline, \getresults 1 aborts commands one at a time.
\startpipeline
-SELECT $1 \bind \g
-SELECT $1 \bind 1 \g
+SELECT $1 \bind \sendpipeline
+SELECT $1 \bind 1 \sendpipeline
SELECT $1 \parse a
-\bind_named a 1 \g
+\bind_named a 1 \sendpipeline
\syncpipeline
\getresults 1
ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
@@ -569,11 +592,11 @@ Pipeline aborted, command did not run
-- Test chunked results with an aborted pipeline.
\startpipeline
\set FETCH_COUNT 10
-SELECT $1 \bind \g
+SELECT $1 \bind \sendpipeline
\flushrequest
\getresults
ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
-SELECT $1 \bind \g
+SELECT $1 \bind \sendpipeline
\endpipeline
fetching results in chunked mode failed
Pipeline aborted, command did not run
@@ -595,7 +618,7 @@ select 1;
-- Error messages accumulate and are repeated.
\startpipeline
-SELECT 1 \bind \g
+SELECT 1 \bind \sendpipeline
SELECT 1;
PQsendQuery not allowed in pipeline mode
SELECT 1;
@@ -616,12 +639,12 @@ PQsendQuery not allowed in pipeline mode
-- commit the implicit transaction block. The first command after a
-- sync will not be seen as belonging to a pipeline.
\startpipeline
-SET LOCAL statement_timeout='1h' \bind \g
-SHOW statement_timeout \bind \g
+SET LOCAL statement_timeout='1h' \bind \sendpipeline
+SHOW statement_timeout \bind \sendpipeline
\syncpipeline
-SHOW statement_timeout \bind \g
-SET LOCAL statement_timeout='2h' \bind \g
-SHOW statement_timeout \bind \g
+SHOW statement_timeout \bind \sendpipeline
+SET LOCAL statement_timeout='2h' \bind \sendpipeline
+SHOW statement_timeout \bind \sendpipeline
\endpipeline
WARNING: SET LOCAL can only be used in transaction blocks
statement_timeout
@@ -641,9 +664,9 @@ WARNING: SET LOCAL can only be used in transaction blocks
-- REINDEX CONCURRENTLY fails if not the first command in a pipeline.
\startpipeline
-SELECT $1 \bind 1 \g
-REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 1 \sendpipeline
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
?column?
----------
@@ -653,8 +676,8 @@ SELECT $1 \bind 2 \g
ERROR: REINDEX CONCURRENTLY cannot run inside a transaction block
-- REINDEX CONCURRENTLY works if it is the first command in a pipeline.
\startpipeline
-REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
-SELECT $1 \bind 2 \g
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
?column?
----------
@@ -663,25 +686,25 @@ SELECT $1 \bind 2 \g
-- Subtransactions are not allowed in a pipeline.
\startpipeline
-SAVEPOINT a \bind \g
-SELECT $1 \bind 1 \g
-ROLLBACK TO SAVEPOINT a \bind \g
-SELECT $1 \bind 2 \g
+SAVEPOINT a \bind \sendpipeline
+SELECT $1 \bind 1 \sendpipeline
+ROLLBACK TO SAVEPOINT a \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
ERROR: SAVEPOINT can only be used in transaction blocks
-- LOCK fails as the first command in a pipeline, as not seen in an
-- implicit transaction block.
\startpipeline
-LOCK psql_pipeline \bind \g
-SELECT $1 \bind 2 \g
+LOCK psql_pipeline \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
ERROR: LOCK TABLE can only be used in transaction blocks
-- LOCK succeeds as it is not the first command in a pipeline,
-- seen in an implicit transaction block.
\startpipeline
-SELECT $1 \bind 1 \g
-LOCK psql_pipeline \bind \g
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 1 \sendpipeline
+LOCK psql_pipeline \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
?column?
----------
@@ -695,12 +718,12 @@ SELECT $1 \bind 2 \g
-- VACUUM works as the first command in a pipeline.
\startpipeline
-VACUUM psql_pipeline \bind \g
+VACUUM psql_pipeline \bind \sendpipeline
\endpipeline
-- VACUUM fails when not the first command in a pipeline.
\startpipeline
-SELECT 1 \bind \g
-VACUUM psql_pipeline \bind \g
+SELECT 1 \bind \sendpipeline
+VACUUM psql_pipeline \bind \sendpipeline
\endpipeline
?column?
----------
@@ -710,9 +733,9 @@ VACUUM psql_pipeline \bind \g
ERROR: VACUUM cannot run inside a transaction block
-- VACUUM works after a \syncpipeline.
\startpipeline
-SELECT 1 \bind \g
+SELECT 1 \bind \sendpipeline
\syncpipeline
-VACUUM psql_pipeline \bind \g
+VACUUM psql_pipeline \bind \sendpipeline
\endpipeline
?column?
----------
diff --git a/src/test/regress/sql/psql.sql b/src/test/regress/sql/psql.sql
index 97d1be3aac3..b6ba947b812 100644
--- a/src/test/regress/sql/psql.sql
+++ b/src/test/regress/sql/psql.sql
@@ -1079,6 +1079,7 @@ select \if false \\ (bogus \else \\ 42 \endif \\ forty_two;
\sf whole_line
\sv whole_line
\startpipeline
+ \sendpipeline
\syncpipeline
\t arg1
\T arg1
diff --git a/src/test/regress/sql/psql_pipeline.sql b/src/test/regress/sql/psql_pipeline.sql
index 6517ebb71f8..256081dfd5f 100644
--- a/src/test/regress/sql/psql_pipeline.sql
+++ b/src/test/regress/sql/psql_pipeline.sql
@@ -6,23 +6,23 @@ CREATE TABLE psql_pipeline(a INTEGER PRIMARY KEY, s TEXT);
-- Single query
\startpipeline
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
-- Multiple queries
\startpipeline
-SELECT $1 \bind 'val1' \g
-SELECT $1, $2 \bind 'val2' 'val3' \g
-SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1 \bind 'val1' \sendpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
\endpipeline
-- Test \flush
\startpipeline
\flush
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\flush
-SELECT $1, $2 \bind 'val2' 'val3' \g
-SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
\endpipeline
-- Send multiple syncs
@@ -30,12 +30,12 @@ SELECT $1, $2 \bind 'val2' 'val3' \g
\echo :PIPELINE_COMMAND_COUNT
\echo :PIPELINE_SYNC_COUNT
\echo :PIPELINE_RESULT_COUNT
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\syncpipeline
\syncpipeline
-SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
\syncpipeline
-SELECT $1, $2 \bind 'val4' 'val5' \g
+SELECT $1, $2 \bind 'val4' 'val5' \sendpipeline
\echo :PIPELINE_COMMAND_COUNT
\echo :PIPELINE_SYNC_COUNT
\echo :PIPELINE_RESULT_COUNT
@@ -44,39 +44,39 @@ SELECT $1, $2 \bind 'val4' 'val5' \g
-- \startpipeline should not have any effect if already in a pipeline.
\startpipeline
\startpipeline
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
-- Convert an implicit transaction block to an explicit transaction block.
\startpipeline
-INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
-BEGIN \bind \g
-INSERT INTO psql_pipeline VALUES ($1) \bind 2 \g
-ROLLBACK \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \sendpipeline
+BEGIN \bind \sendpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 2 \sendpipeline
+ROLLBACK \bind \sendpipeline
\endpipeline
-- Multiple explicit transactions
\startpipeline
-BEGIN \bind \g
-INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
-ROLLBACK \bind \g
-BEGIN \bind \g
-INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
-COMMIT \bind \g
+BEGIN \bind \sendpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \sendpipeline
+ROLLBACK \bind \sendpipeline
+BEGIN \bind \sendpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \sendpipeline
+COMMIT \bind \sendpipeline
\endpipeline
-- COPY FROM STDIN
\startpipeline
-SELECT $1 \bind 'val1' \g
-COPY psql_pipeline FROM STDIN \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+COPY psql_pipeline FROM STDIN \bind \sendpipeline
\endpipeline
2 test2
\.
-- COPY FROM STDIN with \flushrequest + \getresults
\startpipeline
-SELECT $1 \bind 'val1' \g
-COPY psql_pipeline FROM STDIN \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+COPY psql_pipeline FROM STDIN \bind \sendpipeline
\flushrequest
\getresults
3 test3
@@ -85,8 +85,8 @@ COPY psql_pipeline FROM STDIN \bind \g
-- COPY FROM STDIN with \syncpipeline + \getresults
\startpipeline
-SELECT $1 \bind 'val1' \g
-COPY psql_pipeline FROM STDIN \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+COPY psql_pipeline FROM STDIN \bind \sendpipeline
\syncpipeline
\getresults
4 test4
@@ -95,22 +95,22 @@ COPY psql_pipeline FROM STDIN \bind \g
-- COPY TO STDOUT
\startpipeline
-SELECT $1 \bind 'val1' \g
-copy psql_pipeline TO STDOUT \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+copy psql_pipeline TO STDOUT \bind \sendpipeline
\endpipeline
-- COPY TO STDOUT with \flushrequest + \getresults
\startpipeline
-SELECT $1 \bind 'val1' \g
-copy psql_pipeline TO STDOUT \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+copy psql_pipeline TO STDOUT \bind \sendpipeline
\flushrequest
\getresults
\endpipeline
-- COPY TO STDOUT with \syncpipeline + \getresults
\startpipeline
-SELECT $1 \bind 'val1' \g
-copy psql_pipeline TO STDOUT \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+copy psql_pipeline TO STDOUT \bind \sendpipeline
\syncpipeline
\getresults
\endpipeline
@@ -120,22 +120,22 @@ copy psql_pipeline TO STDOUT \bind \g
SELECT $1 \parse ''
SELECT $1, $2 \parse ''
SELECT $2 \parse pipeline_1
-\bind_named '' 1 2 \g
-\bind_named pipeline_1 2 \g
+\bind_named '' 1 2 \sendpipeline
+\bind_named pipeline_1 2 \sendpipeline
\endpipeline
-- \getresults displays all results preceding a \flushrequest.
\startpipeline
-SELECT $1 \bind 1 \g
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 1 \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\flushrequest
\getresults
\endpipeline
-- \getresults displays all results preceding a \syncpipeline.
\startpipeline
-SELECT $1 \bind 1 \g
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 1 \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\syncpipeline
\getresults
\endpipeline
@@ -143,7 +143,7 @@ SELECT $1 \bind 2 \g
-- \getresults immediately returns if there is no result to fetch.
\startpipeline
\getresults
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\getresults
\flushrequest
\endpipeline
@@ -151,42 +151,42 @@ SELECT $1 \bind 2 \g
-- \getresults only fetches results preceding a \flushrequest.
\startpipeline
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\flushrequest
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\getresults
\endpipeline
-- \getresults only fetches results preceding a \syncpipeline.
\startpipeline
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\syncpipeline
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\getresults
\endpipeline
-- Use pipeline with chunked results for both \getresults and \endpipeline.
\startpipeline
\set FETCH_COUNT 10
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\flushrequest
\getresults
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
\unset FETCH_COUNT
-- \getresults with specific number of requested results.
\startpipeline
-SELECT $1 \bind 1 \g
-SELECT $1 \bind 2 \g
-SELECT $1 \bind 3 \g
+SELECT $1 \bind 1 \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
+SELECT $1 \bind 3 \sendpipeline
\echo :PIPELINE_SYNC_COUNT
\syncpipeline
\echo :PIPELINE_SYNC_COUNT
\echo :PIPELINE_RESULT_COUNT
\getresults 1
\echo :PIPELINE_RESULT_COUNT
-SELECT $1 \bind 4 \g
+SELECT $1 \bind 4 \sendpipeline
\getresults 3
\echo :PIPELINE_RESULT_COUNT
\endpipeline
@@ -195,7 +195,7 @@ SELECT $1 \bind 4 \g
\startpipeline
\syncpipeline
\syncpipeline
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
\flushrequest
\getresults 2
\getresults 1
@@ -203,9 +203,9 @@ SELECT $1 \bind 1 \g
-- \getresults 0 should get all the results.
\startpipeline
-SELECT $1 \bind 1 \g
-SELECT $1 \bind 2 \g
-SELECT $1 \bind 3 \g
+SELECT $1 \bind 1 \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
+SELECT $1 \bind 3 \sendpipeline
\syncpipeline
\getresults 0
\endpipeline
@@ -221,36 +221,36 @@ SELECT $1 \bind 3 \g
-- pipeline usable.
\startpipeline
SELECT 1;
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
-- After an aborted pipeline, commands after a \syncpipeline should be
-- displayed.
\startpipeline
-SELECT $1 \bind \g
+SELECT $1 \bind \sendpipeline
\syncpipeline
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
\endpipeline
-- For an incorrect number of parameters, the pipeline is aborted and
-- the following queries will not be executed.
\startpipeline
-SELECT \bind 'val1' \g
-SELECT $1 \bind 'val1' \g
+SELECT \bind 'val1' \sendpipeline
+SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
-- An explicit transaction with an error needs to be rollbacked after
-- the pipeline.
\startpipeline
-BEGIN \bind \g
-INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
-ROLLBACK \bind \g
+BEGIN \bind \sendpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \sendpipeline
+ROLLBACK \bind \sendpipeline
\endpipeline
ROLLBACK;
-- \watch sends a simple query, something not allowed within a pipeline.
\startpipeline
-SELECT \bind \g
+SELECT \bind \sendpipeline
\watch 1
\endpipeline
@@ -258,7 +258,7 @@ SELECT \bind \g
-- and the pipeline should still be usable.
\startpipeline
SELECT $1 \bind 1 \gdesc
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
\endpipeline
-- \gset is not allowed in a pipeline, pipeline should still be usable.
@@ -267,54 +267,72 @@ SELECT $1 as i, $2 as j \parse ''
SELECT $1 as k, $2 as l \parse 'second'
\bind_named '' 1 2 \gset
\bind_named second 1 2 \gset pref02_ \echo :pref02_i :pref02_j
-\bind_named '' 1 2 \g
+\bind_named '' 1 2 \sendpipeline
\endpipeline
--- \gx is not allowed, pipeline should still be usable.
+-- \g and \gx are not allowed, pipeline should still be usable.
\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \g filename
+SELECT $1 \bind 1 \g |cat
+SELECT $1 \bind 1 \g (format=unaligned tuples_only=on)
SELECT $1 \bind 1 \gx
+SELECT $1 \bind 1 \gx filename
+SELECT $1 \bind 1 \gx |cat
+SELECT $1 \bind 1 \gx (format=unaligned tuples_only=on)
\reset
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
\endpipeline
-- \gx warning should be emitted in an aborted pipeline, with
-- pipeline still usable.
\startpipeline
-SELECT $1 \bind \g
+SELECT $1 \bind \sendpipeline
\flushrequest
\getresults
SELECT $1 \bind 1 \gx
\endpipeline
+-- \sendpipeline is not allowed outside of a pipeline
+\sendpipeline
+SELECT $1 \bind 1 \sendpipeline
+\reset
+
+-- \sendpipeline is not allowed if not preceded by \bind or \bind_named
+\startpipeline
+\sendpipeline
+SELECT 1 \sendpipeline
+\endpipeline
+
-- \gexec is not allowed, pipeline should still be usable.
\startpipeline
SELECT 'INSERT INTO psql_pipeline(a) SELECT generate_series(1, 10)' \parse 'insert_stmt'
\bind_named insert_stmt \gexec
-\bind_named insert_stmt \g
+\bind_named insert_stmt \sendpipeline
SELECT COUNT(*) FROM psql_pipeline \bind \g
\endpipeline
-- After an error, pipeline is aborted and requires \syncpipeline to be
-- reusable.
\startpipeline
-SELECT $1 \bind \g
-SELECT $1 \bind 1 \g
+SELECT $1 \bind \sendpipeline
+SELECT $1 \bind 1 \sendpipeline
SELECT $1 \parse a
-\bind_named a 1 \g
+\bind_named a 1 \sendpipeline
\close a
\flushrequest
\getresults
-- Pipeline is aborted.
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
SELECT $1 \parse a
-\bind_named a 1 \g
+\bind_named a 1 \sendpipeline
\close a
-- Sync allows pipeline to recover.
\syncpipeline
\getresults
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
SELECT $1 \parse a
-\bind_named a 1 \g
+\bind_named a 1 \sendpipeline
\close a
\flushrequest
\getresults
@@ -322,10 +340,10 @@ SELECT $1 \parse a
-- In an aborted pipeline, \getresults 1 aborts commands one at a time.
\startpipeline
-SELECT $1 \bind \g
-SELECT $1 \bind 1 \g
+SELECT $1 \bind \sendpipeline
+SELECT $1 \bind 1 \sendpipeline
SELECT $1 \parse a
-\bind_named a 1 \g
+\bind_named a 1 \sendpipeline
\syncpipeline
\getresults 1
\getresults 1
@@ -337,10 +355,10 @@ SELECT $1 \parse a
-- Test chunked results with an aborted pipeline.
\startpipeline
\set FETCH_COUNT 10
-SELECT $1 \bind \g
+SELECT $1 \bind \sendpipeline
\flushrequest
\getresults
-SELECT $1 \bind \g
+SELECT $1 \bind \sendpipeline
\endpipeline
\unset FETCH_COUNT
@@ -356,7 +374,7 @@ select 1;
-- Error messages accumulate and are repeated.
\startpipeline
-SELECT 1 \bind \g
+SELECT 1 \bind \sendpipeline
SELECT 1;
SELECT 1;
\endpipeline
@@ -371,66 +389,66 @@ SELECT 1;
-- commit the implicit transaction block. The first command after a
-- sync will not be seen as belonging to a pipeline.
\startpipeline
-SET LOCAL statement_timeout='1h' \bind \g
-SHOW statement_timeout \bind \g
+SET LOCAL statement_timeout='1h' \bind \sendpipeline
+SHOW statement_timeout \bind \sendpipeline
\syncpipeline
-SHOW statement_timeout \bind \g
-SET LOCAL statement_timeout='2h' \bind \g
-SHOW statement_timeout \bind \g
+SHOW statement_timeout \bind \sendpipeline
+SET LOCAL statement_timeout='2h' \bind \sendpipeline
+SHOW statement_timeout \bind \sendpipeline
\endpipeline
-- REINDEX CONCURRENTLY fails if not the first command in a pipeline.
\startpipeline
-SELECT $1 \bind 1 \g
-REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 1 \sendpipeline
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
-- REINDEX CONCURRENTLY works if it is the first command in a pipeline.
\startpipeline
-REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
-SELECT $1 \bind 2 \g
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
-- Subtransactions are not allowed in a pipeline.
\startpipeline
-SAVEPOINT a \bind \g
-SELECT $1 \bind 1 \g
-ROLLBACK TO SAVEPOINT a \bind \g
-SELECT $1 \bind 2 \g
+SAVEPOINT a \bind \sendpipeline
+SELECT $1 \bind 1 \sendpipeline
+ROLLBACK TO SAVEPOINT a \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
-- LOCK fails as the first command in a pipeline, as not seen in an
-- implicit transaction block.
\startpipeline
-LOCK psql_pipeline \bind \g
-SELECT $1 \bind 2 \g
+LOCK psql_pipeline \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
-- LOCK succeeds as it is not the first command in a pipeline,
-- seen in an implicit transaction block.
\startpipeline
-SELECT $1 \bind 1 \g
-LOCK psql_pipeline \bind \g
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 1 \sendpipeline
+LOCK psql_pipeline \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
-- VACUUM works as the first command in a pipeline.
\startpipeline
-VACUUM psql_pipeline \bind \g
+VACUUM psql_pipeline \bind \sendpipeline
\endpipeline
-- VACUUM fails when not the first command in a pipeline.
\startpipeline
-SELECT 1 \bind \g
-VACUUM psql_pipeline \bind \g
+SELECT 1 \bind \sendpipeline
+VACUUM psql_pipeline \bind \sendpipeline
\endpipeline
-- VACUUM works after a \syncpipeline.
\startpipeline
-SELECT 1 \bind \g
+SELECT 1 \bind \sendpipeline
\syncpipeline
-VACUUM psql_pipeline \bind \g
+VACUUM psql_pipeline \bind \sendpipeline
\endpipeline
-- Clean up
--
2.39.5 (Apple Git-154)
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-04 23:11 Michael Paquier <[email protected]>
parent: Anthonin Bonnefoy <[email protected]>
0 siblings, 0 replies; 41+ messages in thread
From: Michael Paquier @ 2025-03-04 23:11 UTC (permalink / raw)
To: Anthonin Bonnefoy <[email protected]>; +Cc: Daniel Verite <[email protected]>; Jelte Fennema-Nio <[email protected]>; pgsql-hackers
On Tue, Mar 04, 2025 at 10:31:46AM +0100, Anthonin Bonnefoy wrote:
> Saving the printQueryOpt when a command is pushed was an option I had
> in mind if that was straightforward to implement. However, even with
> savePsetInfo, you will need to save values like gfname and gset_prefix
> since it impacts the output (it may make sense to move those in
> printQueryOpt). This would also need to be saved for all commands,
> like \close or \parse since we don't distinguish if a piped command
> generates an output or not. So that definitely looks like it would add
> a lot of complexity for limited benefit.
Yeah, same opinion here. I don't want this level of complexity with
extra manipulations of printQueryOpt when fetching the results,
either. I'm all for making these meta-commands to what we think is
more natural, but not at the cost of a more complex logic in the
result printing depending on what's been given by a meta-command when
a query is pushed to a pipeline.
> I took a stab at creating the \sendpipeline meta-command. I've also
> realised there's a small leak where fname is currently not freed on
> queries like 'select ... \bind \gx filename' when within a pipeline,
> which is fixed in patch 0001.
Indeed. Fixed this one for now.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-07 00:05 Jelte Fennema-Nio <[email protected]>
parent: Michael Paquier <[email protected]>
1 sibling, 2 replies; 41+ messages in thread
From: Jelte Fennema-Nio @ 2025-03-07 00:05 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Anthonin Bonnefoy <[email protected]>; pgsql-hackers
On Tue, 25 Feb 2025 at 02:11, Michael Paquier <[email protected]> wrote:
> Initial digestion has gone well.
One thing I've noticed is that \startpipeline throws warnings when
copy pasting multiple lines. It seems to still execute everything as
expected though. As an example you can copy paste this tiny script:
\startpipeline
select pg_sleep(5) \bind \g
\endpipeline
And then it will show these "extra argument ... ignored" warnings
\startpipeline: extra argument "select" ignored
\startpipeline: extra argument "pg_sleep(5)" ignored
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-07 08:08 Anthonin Bonnefoy <[email protected]>
parent: Jelte Fennema-Nio <[email protected]>
1 sibling, 0 replies; 41+ messages in thread
From: Anthonin Bonnefoy @ 2025-03-07 08:08 UTC (permalink / raw)
To: Jelte Fennema-Nio <[email protected]>; +Cc: Michael Paquier <[email protected]>; pgsql-hackers
On Thu, Mar 6, 2025 at 5:20 AM Michael Paquier <[email protected]> wrote:
> That was not a test case we had in mind originally here, but if it is
> possible to keep the implementation simple while supporting your
> demand, well, let's do it. If it's not that straight-forward, let's
> use the new meta-command, forbidding \g and \gx based on your
> arguments from upthread.
I think the new meta-command is a separate issue from allowing ';' to
push in a pipeline. Any time there's a change or an additional format
option added to \g, it will need to be forbidden for pipelining. The
\sendpipeline meta-command will help keep those exceptions low since
the whole \g will be forbidden.
Another possible option would be to allow both \g and \gx, but send a
warning like "printing options within a pipeline will be ignored" if
those options are used, similar to "SET LOCAL" warning when done
outside of a transaction block. That would have the benefit of making
existing scripts using \g and \gx compatible.
For using ';' to push commands in a pipeline, I think it should be
fairly straightforward. I can try to work on that next week (I'm
currently chasing a weird memory context bug that I need to finish
first).
On Fri, Mar 7, 2025 at 1:05 AM Jelte Fennema-Nio <[email protected]> wrote:
> One thing I've noticed is that \startpipeline throws warnings when
> copy pasting multiple lines. It seems to still execute everything as
> expected though. As an example you can copy paste this tiny script:
>
> \startpipeline
> select pg_sleep(5) \bind \g
> \endpipeline
>
> And then it will show these "extra argument ... ignored" warnings
>
> \startpipeline: extra argument "select" ignored
> \startpipeline: extra argument "pg_sleep(5)" ignored
It looks like an issue with libreadline. At least, I've been able to
reproduce the warnings and 'readline(prompt);' returns everything as a
single line, with the \n inside the string. This explains why what is
after \startpipeline is processed as arguments. This can also be done
with:
select 1 \bind \g
select 2 \bind \g
And somehow, I couldn't reproduce the issue anymore once I've compiled
and installed libreadline with debug symbols.
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-07 22:31 Daniel Verite <[email protected]>
parent: Jelte Fennema-Nio <[email protected]>
1 sibling, 1 reply; 41+ messages in thread
From: Daniel Verite @ 2025-03-07 22:31 UTC (permalink / raw)
To: Jelte Fennema-Nio <[email protected]>; +Cc: Michael Paquier <[email protected]>; Anthonin Bonnefoy <[email protected]>; pgsql-hackers
Jelte Fennema-Nio wrote:
> As an example you can copy paste this tiny script:
>
> \startpipeline
> select pg_sleep(5) \bind \g
> \endpipeline
>
> And then it will show these "extra argument ... ignored" warnings
>
> \startpipeline: extra argument "select" ignored
> \startpipeline: extra argument "pg_sleep(5)" ignored
It happens with other metacommands as well, and appears
to depend on a readline option that is "on" by default since
readline-8.1 [1]
enable-bracketed-paste
When set to ‘On’, Readline configures the terminal to insert each
paste into the editing buffer as a single string of characters,
instead of treating each character as if it had been read from the
keyboard. This is called putting the terminal into bracketed paste
mode; it prevents Readline from executing any editing commands
bound to key sequences appearing in the pasted text. The default
is ‘On’.
This behavior of the metacommand complaining about arguments
on the next line also happens if using \e and typing this sequence
of commands in the editor. In that case readline is not involved.
There might be something to improve here, because
a metacommand cannot take its argument from the next line,
and yet that's what the error messages somewhat imply.
But that issue is not related to the new pipeline metacommands.
[1]
https://tiswww.case.edu/php/chet/readline/readline.html#index-enable_002dbracketed_002dpaste
Best regards,
--
Daniel Vérité
https://postgresql.verite.pro/
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-17 09:50 Anthonin Bonnefoy <[email protected]>
parent: Daniel Verite <[email protected]>
0 siblings, 3 replies; 41+ messages in thread
From: Anthonin Bonnefoy @ 2025-03-17 09:50 UTC (permalink / raw)
To: Daniel Verite <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; Michael Paquier <[email protected]>; pgsql-hackers
Here is a new patch set:
0001: This introduces the \sendpipeline meta-command and forbid \g in
a pipeline. This is to fix the formatting options of \g that are not
supported in a pipeline.
0002: Allows ';' to send a query using extended protocol when within a
pipeline by using PQsendQueryParams with 0 parameters. It is not
possible to send parameters with extended protocol this way and
everything will be propagated through the query string, similar to a
simple query.
Attachments:
[application/octet-stream] v02-0001-psql-Create-new-sendpipeline-meta-command.patch (38.8K, ../../CAO6_Xqq4JEHRaGyC=r+dpi0Ejbp2P3aUo-KfR2NuipCvE-o_ZA@mail.gmail.com/2-v02-0001-psql-Create-new-sendpipeline-meta-command.patch)
download | inline diff:
From 097084856575dbcd53baad01a1b5420b4201036d Mon Sep 17 00:00:00 2001
From: Anthonin Bonnefoy <[email protected]>
Date: Tue, 4 Mar 2025 09:47:45 +0100
Subject: psql: Create new \sendpipeline meta-command
In the initial pipelining support for psql, \g was reused as a way to
push extended query into an ongoing pipeline. However, all related
meta-command options are not applicable in pipeline mode. Pipeline
output only happens during \getresults or \endpipeline and any change to
output options would have been reset.
To avoid possible confusion, a new \sendpipeline meta-command is
introduced, replacing \g when within a pipeline.
---
doc/src/sgml/ref/psql-ref.sgml | 11 +-
src/bin/psql/command.c | 45 +++-
src/bin/psql/help.c | 1 +
src/bin/psql/tab-complete.in.c | 2 +-
src/test/regress/expected/psql.out | 1 +
src/test/regress/expected/psql_pipeline.out | 245 +++++++++++---------
src/test/regress/sql/psql.sql | 1 +
src/test/regress/sql/psql_pipeline.sql | 230 +++++++++---------
8 files changed, 310 insertions(+), 226 deletions(-)
diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml
index cedccc14129..cddf6e07531 100644
--- a/doc/src/sgml/ref/psql-ref.sgml
+++ b/doc/src/sgml/ref/psql-ref.sgml
@@ -3677,6 +3677,7 @@ testdb=> <userinput>\setenv LESS -imx4F</userinput>
<varlistentry id="app-psql-meta-command-pipeline">
<term><literal>\startpipeline</literal></term>
+ <term><literal>\sendpipeline</literal></term>
<term><literal>\syncpipeline</literal></term>
<term><literal>\endpipeline</literal></term>
<term><literal>\flushrequest</literal></term>
@@ -3701,10 +3702,10 @@ testdb=> <userinput>\setenv LESS -imx4F</userinput>
queries need to be sent using the meta-commands
<literal>\bind</literal>, <literal>\bind_named</literal>,
<literal>\close</literal> or <literal>\parse</literal>. While a
- pipeline is ongoing, <literal>\g</literal> will append the current
- query buffer to the pipeline. Other meta-commands like
- <literal>\gx</literal> or <literal>\gdesc</literal> are not allowed
- in pipeline mode.
+ pipeline is ongoing, <literal>\sendpipeline</literal> will append the
+ current query buffer to the pipeline. Other meta-commands like
+ <literal>\g</literal>, <literal>\gx</literal> or <literal>\gdesc</literal>
+ are not allowed in pipeline mode.
</para>
<para>
@@ -3738,7 +3739,7 @@ testdb=> <userinput>\setenv LESS -imx4F</userinput>
Example:
<programlisting>
\startpipeline
-SELECT 1 \bind \g
+SELECT 1 \bind \sendpipeline
\flushrequest
\getresults
\endpipeline
diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c
index fb0b27568c5..7670c20b29e 100644
--- a/src/bin/psql/command.c
+++ b/src/bin/psql/command.c
@@ -137,6 +137,7 @@ static backslashResult exec_command_setenv(PsqlScanState scan_state, bool active
static backslashResult exec_command_sf_sv(PsqlScanState scan_state, bool active_branch,
const char *cmd, bool is_func);
static backslashResult exec_command_startpipeline(PsqlScanState scan_state, bool active_branch);
+static backslashResult exec_command_sendpipeline(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_syncpipeline(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_t(PsqlScanState scan_state, bool active_branch);
static backslashResult exec_command_T(PsqlScanState scan_state, bool active_branch);
@@ -427,6 +428,8 @@ exec_command(const char *cmd,
status = exec_command_sf_sv(scan_state, active_branch, cmd, false);
else if (strcmp(cmd, "startpipeline") == 0)
status = exec_command_startpipeline(scan_state, active_branch);
+ else if (strcmp(cmd, "sendpipeline") == 0)
+ status = exec_command_sendpipeline(scan_state, active_branch);
else if (strcmp(cmd, "syncpipeline") == 0)
status = exec_command_syncpipeline(scan_state, active_branch);
else if (strcmp(cmd, "t") == 0)
@@ -1734,10 +1737,9 @@ exec_command_g(PsqlScanState scan_state, bool active_branch, const char *cmd)
if (status == PSQL_CMD_SKIP_LINE && active_branch)
{
- if (strcmp(cmd, "gx") == 0 &&
- PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
{
- pg_log_error("\\gx not allowed in pipeline mode");
+ pg_log_error("\\%s not allowed in pipeline mode", cmd);
clean_extended_state();
free(fname);
return PSQL_CMD_ERROR;
@@ -2979,6 +2981,43 @@ exec_command_startpipeline(PsqlScanState scan_state, bool active_branch)
return status;
}
+/*
+ * \sendpipeline -- send an extended query to an ongoing pipeline
+ */
+static backslashResult
+exec_command_sendpipeline(PsqlScanState scan_state, bool active_branch)
+{
+ backslashResult status = PSQL_CMD_SKIP_LINE;
+
+ if (active_branch)
+ {
+ if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ if (pset.send_mode == PSQL_SEND_EXTENDED_QUERY_PREPARED ||
+ pset.send_mode == PSQL_SEND_EXTENDED_QUERY_PARAMS)
+ {
+ status = PSQL_CMD_SEND;
+ }
+ else
+ {
+ pg_log_error("\\sendpipeline must be used after \\bind or \\bind_named");
+ clean_extended_state();
+ return PSQL_CMD_ERROR;
+ }
+ }
+ else
+ {
+ pg_log_error("\\sendpipeline not allowed outside of pipeline mode");
+ clean_extended_state();
+ return PSQL_CMD_ERROR;
+ }
+ }
+ else
+ ignore_slash_options(scan_state);
+
+ return status;
+}
+
/*
* \syncpipeline -- send a sync message to an active pipeline
*/
diff --git a/src/bin/psql/help.c b/src/bin/psql/help.c
index 714b8619233..edfc794b76f 100644
--- a/src/bin/psql/help.c
+++ b/src/bin/psql/help.c
@@ -182,6 +182,7 @@ slashUsage(unsigned short int pager)
HELP0(" \\parse STMT_NAME create a prepared statement\n");
HELP0(" \\q quit psql\n");
HELP0(" \\startpipeline enter pipeline mode\n");
+ HELP0(" \\sendpipeline send an extended query to an ongoing pipeline\n");
HELP0(" \\syncpipeline add a synchronisation point to an ongoing pipeline\n");
HELP0(" \\watch [[i=]SEC] [c=N] [m=MIN]\n"
" execute query every SEC seconds, up to N times,\n"
diff --git a/src/bin/psql/tab-complete.in.c b/src/bin/psql/tab-complete.in.c
index 8432be641ac..b2de63a5b74 100644
--- a/src/bin/psql/tab-complete.in.c
+++ b/src/bin/psql/tab-complete.in.c
@@ -1895,7 +1895,7 @@ psql_completion(const char *text, int start, int end)
"\\parse", "\\password", "\\print", "\\prompt", "\\pset",
"\\qecho", "\\quit",
"\\reset",
- "\\s", "\\set", "\\setenv", "\\sf", "\\startpipeline", "\\sv", "\\syncpipeline",
+ "\\s", "\\set", "\\setenv", "\\sf", "\\startpipeline", "\\sendpipeline", "\\sv", "\\syncpipeline",
"\\t", "\\T", "\\timing",
"\\unset",
"\\x",
diff --git a/src/test/regress/expected/psql.out b/src/test/regress/expected/psql.out
index 6543e90de75..ee6f8671bea 100644
--- a/src/test/regress/expected/psql.out
+++ b/src/test/regress/expected/psql.out
@@ -4711,6 +4711,7 @@ invalid command \lo
\sf whole_line
\sv whole_line
\startpipeline
+ \sendpipeline
\syncpipeline
\t arg1
\T arg1
diff --git a/src/test/regress/expected/psql_pipeline.out b/src/test/regress/expected/psql_pipeline.out
index 3df2415a840..53f2edfd986 100644
--- a/src/test/regress/expected/psql_pipeline.out
+++ b/src/test/regress/expected/psql_pipeline.out
@@ -4,7 +4,7 @@
CREATE TABLE psql_pipeline(a INTEGER PRIMARY KEY, s TEXT);
-- Single query
\startpipeline
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
?column?
----------
@@ -13,9 +13,9 @@ SELECT $1 \bind 'val1' \g
-- Multiple queries
\startpipeline
-SELECT $1 \bind 'val1' \g
-SELECT $1, $2 \bind 'val2' 'val3' \g
-SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1 \bind 'val1' \sendpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
\endpipeline
?column?
----------
@@ -35,10 +35,10 @@ SELECT $1, $2 \bind 'val2' 'val3' \g
-- Test \flush
\startpipeline
\flush
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\flush
-SELECT $1, $2 \bind 'val2' 'val3' \g
-SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
\endpipeline
?column?
----------
@@ -63,12 +63,12 @@ SELECT $1, $2 \bind 'val2' 'val3' \g
0
\echo :PIPELINE_RESULT_COUNT
0
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\syncpipeline
\syncpipeline
-SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
\syncpipeline
-SELECT $1, $2 \bind 'val4' 'val5' \g
+SELECT $1, $2 \bind 'val4' 'val5' \sendpipeline
\echo :PIPELINE_COMMAND_COUNT
1
\echo :PIPELINE_SYNC_COUNT
@@ -94,7 +94,7 @@ SELECT $1, $2 \bind 'val4' 'val5' \g
-- \startpipeline should not have any effect if already in a pipeline.
\startpipeline
\startpipeline
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
?column?
----------
@@ -103,24 +103,24 @@ SELECT $1 \bind 'val1' \g
-- Convert an implicit transaction block to an explicit transaction block.
\startpipeline
-INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
-BEGIN \bind \g
-INSERT INTO psql_pipeline VALUES ($1) \bind 2 \g
-ROLLBACK \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \sendpipeline
+BEGIN \bind \sendpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 2 \sendpipeline
+ROLLBACK \bind \sendpipeline
\endpipeline
-- Multiple explicit transactions
\startpipeline
-BEGIN \bind \g
-INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
-ROLLBACK \bind \g
-BEGIN \bind \g
-INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
-COMMIT \bind \g
+BEGIN \bind \sendpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \sendpipeline
+ROLLBACK \bind \sendpipeline
+BEGIN \bind \sendpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \sendpipeline
+COMMIT \bind \sendpipeline
\endpipeline
-- COPY FROM STDIN
\startpipeline
-SELECT $1 \bind 'val1' \g
-COPY psql_pipeline FROM STDIN \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+COPY psql_pipeline FROM STDIN \bind \sendpipeline
\endpipeline
?column?
----------
@@ -129,8 +129,8 @@ COPY psql_pipeline FROM STDIN \bind \g
-- COPY FROM STDIN with \flushrequest + \getresults
\startpipeline
-SELECT $1 \bind 'val1' \g
-COPY psql_pipeline FROM STDIN \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+COPY psql_pipeline FROM STDIN \bind \sendpipeline
\flushrequest
\getresults
?column?
@@ -142,8 +142,8 @@ message type 0x5a arrived from server while idle
\endpipeline
-- COPY FROM STDIN with \syncpipeline + \getresults
\startpipeline
-SELECT $1 \bind 'val1' \g
-COPY psql_pipeline FROM STDIN \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+COPY psql_pipeline FROM STDIN \bind \sendpipeline
\syncpipeline
\getresults
?column?
@@ -154,8 +154,8 @@ COPY psql_pipeline FROM STDIN \bind \g
\endpipeline
-- COPY TO STDOUT
\startpipeline
-SELECT $1 \bind 'val1' \g
-copy psql_pipeline TO STDOUT \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+copy psql_pipeline TO STDOUT \bind \sendpipeline
\endpipeline
?column?
----------
@@ -168,8 +168,8 @@ copy psql_pipeline TO STDOUT \bind \g
4 test4
-- COPY TO STDOUT with \flushrequest + \getresults
\startpipeline
-SELECT $1 \bind 'val1' \g
-copy psql_pipeline TO STDOUT \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+copy psql_pipeline TO STDOUT \bind \sendpipeline
\flushrequest
\getresults
?column?
@@ -184,8 +184,8 @@ copy psql_pipeline TO STDOUT \bind \g
\endpipeline
-- COPY TO STDOUT with \syncpipeline + \getresults
\startpipeline
-SELECT $1 \bind 'val1' \g
-copy psql_pipeline TO STDOUT \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+copy psql_pipeline TO STDOUT \bind \sendpipeline
\syncpipeline
\getresults
?column?
@@ -203,14 +203,14 @@ copy psql_pipeline TO STDOUT \bind \g
SELECT $1 \parse ''
SELECT $1, $2 \parse ''
SELECT $2 \parse pipeline_1
-\bind_named '' 1 2 \g
-\bind_named pipeline_1 2 \g
+\bind_named '' 1 2 \sendpipeline
+\bind_named pipeline_1 2 \sendpipeline
\endpipeline
ERROR: could not determine data type of parameter $1
-- \getresults displays all results preceding a \flushrequest.
\startpipeline
-SELECT $1 \bind 1 \g
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 1 \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\flushrequest
\getresults
?column?
@@ -226,8 +226,8 @@ SELECT $1 \bind 2 \g
\endpipeline
-- \getresults displays all results preceding a \syncpipeline.
\startpipeline
-SELECT $1 \bind 1 \g
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 1 \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\syncpipeline
\getresults
?column?
@@ -245,7 +245,7 @@ SELECT $1 \bind 2 \g
\startpipeline
\getresults
No pending results to get
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\getresults
No pending results to get
\flushrequest
@@ -259,9 +259,9 @@ No pending results to get
No pending results to get
-- \getresults only fetches results preceding a \flushrequest.
\startpipeline
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\flushrequest
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\getresults
?column?
----------
@@ -276,9 +276,9 @@ SELECT $1 \bind 2 \g
-- \getresults only fetches results preceding a \syncpipeline.
\startpipeline
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\syncpipeline
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\getresults
?column?
----------
@@ -294,7 +294,7 @@ SELECT $1 \bind 2 \g
-- Use pipeline with chunked results for both \getresults and \endpipeline.
\startpipeline
\set FETCH_COUNT 10
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\flushrequest
\getresults
?column?
@@ -302,7 +302,7 @@ SELECT $1 \bind 2 \g
2
(1 row)
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
?column?
----------
@@ -312,9 +312,9 @@ SELECT $1 \bind 2 \g
\unset FETCH_COUNT
-- \getresults with specific number of requested results.
\startpipeline
-SELECT $1 \bind 1 \g
-SELECT $1 \bind 2 \g
-SELECT $1 \bind 3 \g
+SELECT $1 \bind 1 \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
+SELECT $1 \bind 3 \sendpipeline
\echo :PIPELINE_SYNC_COUNT
0
\syncpipeline
@@ -330,7 +330,7 @@ SELECT $1 \bind 3 \g
\echo :PIPELINE_RESULT_COUNT
2
-SELECT $1 \bind 4 \g
+SELECT $1 \bind 4 \sendpipeline
\getresults 3
?column?
----------
@@ -354,7 +354,7 @@ SELECT $1 \bind 4 \g
\startpipeline
\syncpipeline
\syncpipeline
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
\flushrequest
\getresults 2
\getresults 1
@@ -366,9 +366,9 @@ SELECT $1 \bind 1 \g
\endpipeline
-- \getresults 0 should get all the results.
\startpipeline
-SELECT $1 \bind 1 \g
-SELECT $1 \bind 2 \g
-SELECT $1 \bind 3 \g
+SELECT $1 \bind 1 \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
+SELECT $1 \bind 3 \sendpipeline
\syncpipeline
\getresults 0
?column?
@@ -398,7 +398,7 @@ cannot send pipeline when not in pipeline mode
\startpipeline
SELECT 1;
PQsendQuery not allowed in pipeline mode
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
?column?
----------
@@ -408,9 +408,9 @@ SELECT $1 \bind 'val1' \g
-- After an aborted pipeline, commands after a \syncpipeline should be
-- displayed.
\startpipeline
-SELECT $1 \bind \g
+SELECT $1 \bind \sendpipeline
\syncpipeline
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
\endpipeline
ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
?column?
@@ -421,23 +421,23 @@ ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
-- For an incorrect number of parameters, the pipeline is aborted and
-- the following queries will not be executed.
\startpipeline
-SELECT \bind 'val1' \g
-SELECT $1 \bind 'val1' \g
+SELECT \bind 'val1' \sendpipeline
+SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
ERROR: bind message supplies 1 parameters, but prepared statement "" requires 0
-- An explicit transaction with an error needs to be rollbacked after
-- the pipeline.
\startpipeline
-BEGIN \bind \g
-INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
-ROLLBACK \bind \g
+BEGIN \bind \sendpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \sendpipeline
+ROLLBACK \bind \sendpipeline
\endpipeline
ERROR: duplicate key value violates unique constraint "psql_pipeline_pkey"
DETAIL: Key (a)=(1) already exists.
ROLLBACK;
-- \watch sends a simple query, something not allowed within a pipeline.
\startpipeline
-SELECT \bind \g
+SELECT \bind \sendpipeline
\watch 1
PQsendQuery not allowed in pipeline mode
@@ -450,7 +450,7 @@ PQsendQuery not allowed in pipeline mode
\startpipeline
SELECT $1 \bind 1 \gdesc
synchronous command execution functions are not allowed in pipeline mode
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
\endpipeline
?column?
----------
@@ -465,19 +465,33 @@ SELECT $1 as k, $2 as l \parse 'second'
\gset not allowed in pipeline mode
\bind_named second 1 2 \gset pref02_ \echo :pref02_i :pref02_j
\gset not allowed in pipeline mode
-\bind_named '' 1 2 \g
+\bind_named '' 1 2 \sendpipeline
\endpipeline
i | j
---+---
1 | 2
(1 row)
--- \gx is not allowed, pipeline should still be usable.
+-- \g and \gx are not allowed, pipeline should still be usable.
\startpipeline
+SELECT $1 \bind 1 \g
+\g not allowed in pipeline mode
+SELECT $1 \bind 1 \g filename
+\g not allowed in pipeline mode
+SELECT $1 \bind 1 \g |cat
+\g not allowed in pipeline mode
+SELECT $1 \bind 1 \g (format=unaligned tuples_only=on)
+\g not allowed in pipeline mode
SELECT $1 \bind 1 \gx
\gx not allowed in pipeline mode
+SELECT $1 \bind 1 \gx filename
+\gx not allowed in pipeline mode
+SELECT $1 \bind 1 \gx |cat
+\gx not allowed in pipeline mode
+SELECT $1 \bind 1 \gx (format=unaligned tuples_only=on)
+\gx not allowed in pipeline mode
\reset
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
\endpipeline
?column?
----------
@@ -487,54 +501,63 @@ SELECT $1 \bind 1 \g
-- \gx warning should be emitted in an aborted pipeline, with
-- pipeline still usable.
\startpipeline
-SELECT $1 \bind \g
+SELECT $1 \bind \sendpipeline
\flushrequest
\getresults
ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
SELECT $1 \bind 1 \gx
\gx not allowed in pipeline mode
\endpipeline
+-- \sendpipeline is not allowed outside of a pipeline
+\sendpipeline
+\sendpipeline not allowed outside of pipeline mode
+SELECT $1 \bind 1 \sendpipeline
+\sendpipeline not allowed outside of pipeline mode
+\reset
+-- \sendpipeline is not allowed if not preceded by \bind or \bind_named
+\startpipeline
+\sendpipeline
+\sendpipeline must be used after \bind or \bind_named
+SELECT 1 \sendpipeline
+\sendpipeline must be used after \bind or \bind_named
+\endpipeline
-- \gexec is not allowed, pipeline should still be usable.
\startpipeline
SELECT 'INSERT INTO psql_pipeline(a) SELECT generate_series(1, 10)' \parse 'insert_stmt'
\bind_named insert_stmt \gexec
\gexec not allowed in pipeline mode
-\bind_named insert_stmt \g
+\bind_named insert_stmt \sendpipeline
SELECT COUNT(*) FROM psql_pipeline \bind \g
+\g not allowed in pipeline mode
\endpipeline
?column?
------------------------------------------------------------
INSERT INTO psql_pipeline(a) SELECT generate_series(1, 10)
(1 row)
- count
--------
- 4
-(1 row)
-
-- After an error, pipeline is aborted and requires \syncpipeline to be
-- reusable.
\startpipeline
-SELECT $1 \bind \g
-SELECT $1 \bind 1 \g
+SELECT $1 \bind \sendpipeline
+SELECT $1 \bind 1 \sendpipeline
SELECT $1 \parse a
-\bind_named a 1 \g
+\bind_named a 1 \sendpipeline
\close a
\flushrequest
\getresults
ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
-- Pipeline is aborted.
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
SELECT $1 \parse a
-\bind_named a 1 \g
+\bind_named a 1 \sendpipeline
\close a
-- Sync allows pipeline to recover.
\syncpipeline
\getresults
Pipeline aborted, command did not run
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
SELECT $1 \parse a
-\bind_named a 1 \g
+\bind_named a 1 \sendpipeline
\close a
\flushrequest
\getresults
@@ -551,10 +574,10 @@ SELECT $1 \parse a
\endpipeline
-- In an aborted pipeline, \getresults 1 aborts commands one at a time.
\startpipeline
-SELECT $1 \bind \g
-SELECT $1 \bind 1 \g
+SELECT $1 \bind \sendpipeline
+SELECT $1 \bind 1 \sendpipeline
SELECT $1 \parse a
-\bind_named a 1 \g
+\bind_named a 1 \sendpipeline
\syncpipeline
\getresults 1
ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
@@ -569,11 +592,11 @@ Pipeline aborted, command did not run
-- Test chunked results with an aborted pipeline.
\startpipeline
\set FETCH_COUNT 10
-SELECT $1 \bind \g
+SELECT $1 \bind \sendpipeline
\flushrequest
\getresults
ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
-SELECT $1 \bind \g
+SELECT $1 \bind \sendpipeline
\endpipeline
fetching results in chunked mode failed
Pipeline aborted, command did not run
@@ -595,7 +618,7 @@ select 1;
-- Error messages accumulate and are repeated.
\startpipeline
-SELECT 1 \bind \g
+SELECT 1 \bind \sendpipeline
SELECT 1;
PQsendQuery not allowed in pipeline mode
SELECT 1;
@@ -616,12 +639,12 @@ PQsendQuery not allowed in pipeline mode
-- commit the implicit transaction block. The first command after a
-- sync will not be seen as belonging to a pipeline.
\startpipeline
-SET LOCAL statement_timeout='1h' \bind \g
-SHOW statement_timeout \bind \g
+SET LOCAL statement_timeout='1h' \bind \sendpipeline
+SHOW statement_timeout \bind \sendpipeline
\syncpipeline
-SHOW statement_timeout \bind \g
-SET LOCAL statement_timeout='2h' \bind \g
-SHOW statement_timeout \bind \g
+SHOW statement_timeout \bind \sendpipeline
+SET LOCAL statement_timeout='2h' \bind \sendpipeline
+SHOW statement_timeout \bind \sendpipeline
\endpipeline
WARNING: SET LOCAL can only be used in transaction blocks
statement_timeout
@@ -641,9 +664,9 @@ WARNING: SET LOCAL can only be used in transaction blocks
-- REINDEX CONCURRENTLY fails if not the first command in a pipeline.
\startpipeline
-SELECT $1 \bind 1 \g
-REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 1 \sendpipeline
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
?column?
----------
@@ -653,8 +676,8 @@ SELECT $1 \bind 2 \g
ERROR: REINDEX CONCURRENTLY cannot run inside a transaction block
-- REINDEX CONCURRENTLY works if it is the first command in a pipeline.
\startpipeline
-REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
-SELECT $1 \bind 2 \g
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
?column?
----------
@@ -663,25 +686,25 @@ SELECT $1 \bind 2 \g
-- Subtransactions are not allowed in a pipeline.
\startpipeline
-SAVEPOINT a \bind \g
-SELECT $1 \bind 1 \g
-ROLLBACK TO SAVEPOINT a \bind \g
-SELECT $1 \bind 2 \g
+SAVEPOINT a \bind \sendpipeline
+SELECT $1 \bind 1 \sendpipeline
+ROLLBACK TO SAVEPOINT a \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
ERROR: SAVEPOINT can only be used in transaction blocks
-- LOCK fails as the first command in a pipeline, as not seen in an
-- implicit transaction block.
\startpipeline
-LOCK psql_pipeline \bind \g
-SELECT $1 \bind 2 \g
+LOCK psql_pipeline \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
ERROR: LOCK TABLE can only be used in transaction blocks
-- LOCK succeeds as it is not the first command in a pipeline,
-- seen in an implicit transaction block.
\startpipeline
-SELECT $1 \bind 1 \g
-LOCK psql_pipeline \bind \g
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 1 \sendpipeline
+LOCK psql_pipeline \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
?column?
----------
@@ -695,12 +718,12 @@ SELECT $1 \bind 2 \g
-- VACUUM works as the first command in a pipeline.
\startpipeline
-VACUUM psql_pipeline \bind \g
+VACUUM psql_pipeline \bind \sendpipeline
\endpipeline
-- VACUUM fails when not the first command in a pipeline.
\startpipeline
-SELECT 1 \bind \g
-VACUUM psql_pipeline \bind \g
+SELECT 1 \bind \sendpipeline
+VACUUM psql_pipeline \bind \sendpipeline
\endpipeline
?column?
----------
@@ -710,9 +733,9 @@ VACUUM psql_pipeline \bind \g
ERROR: VACUUM cannot run inside a transaction block
-- VACUUM works after a \syncpipeline.
\startpipeline
-SELECT 1 \bind \g
+SELECT 1 \bind \sendpipeline
\syncpipeline
-VACUUM psql_pipeline \bind \g
+VACUUM psql_pipeline \bind \sendpipeline
\endpipeline
?column?
----------
diff --git a/src/test/regress/sql/psql.sql b/src/test/regress/sql/psql.sql
index 97d1be3aac3..b6ba947b812 100644
--- a/src/test/regress/sql/psql.sql
+++ b/src/test/regress/sql/psql.sql
@@ -1079,6 +1079,7 @@ select \if false \\ (bogus \else \\ 42 \endif \\ forty_two;
\sf whole_line
\sv whole_line
\startpipeline
+ \sendpipeline
\syncpipeline
\t arg1
\T arg1
diff --git a/src/test/regress/sql/psql_pipeline.sql b/src/test/regress/sql/psql_pipeline.sql
index 6517ebb71f8..256081dfd5f 100644
--- a/src/test/regress/sql/psql_pipeline.sql
+++ b/src/test/regress/sql/psql_pipeline.sql
@@ -6,23 +6,23 @@ CREATE TABLE psql_pipeline(a INTEGER PRIMARY KEY, s TEXT);
-- Single query
\startpipeline
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
-- Multiple queries
\startpipeline
-SELECT $1 \bind 'val1' \g
-SELECT $1, $2 \bind 'val2' 'val3' \g
-SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1 \bind 'val1' \sendpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
\endpipeline
-- Test \flush
\startpipeline
\flush
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\flush
-SELECT $1, $2 \bind 'val2' 'val3' \g
-SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
\endpipeline
-- Send multiple syncs
@@ -30,12 +30,12 @@ SELECT $1, $2 \bind 'val2' 'val3' \g
\echo :PIPELINE_COMMAND_COUNT
\echo :PIPELINE_SYNC_COUNT
\echo :PIPELINE_RESULT_COUNT
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\syncpipeline
\syncpipeline
-SELECT $1, $2 \bind 'val2' 'val3' \g
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
\syncpipeline
-SELECT $1, $2 \bind 'val4' 'val5' \g
+SELECT $1, $2 \bind 'val4' 'val5' \sendpipeline
\echo :PIPELINE_COMMAND_COUNT
\echo :PIPELINE_SYNC_COUNT
\echo :PIPELINE_RESULT_COUNT
@@ -44,39 +44,39 @@ SELECT $1, $2 \bind 'val4' 'val5' \g
-- \startpipeline should not have any effect if already in a pipeline.
\startpipeline
\startpipeline
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
-- Convert an implicit transaction block to an explicit transaction block.
\startpipeline
-INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
-BEGIN \bind \g
-INSERT INTO psql_pipeline VALUES ($1) \bind 2 \g
-ROLLBACK \bind \g
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \sendpipeline
+BEGIN \bind \sendpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 2 \sendpipeline
+ROLLBACK \bind \sendpipeline
\endpipeline
-- Multiple explicit transactions
\startpipeline
-BEGIN \bind \g
-INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
-ROLLBACK \bind \g
-BEGIN \bind \g
-INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
-COMMIT \bind \g
+BEGIN \bind \sendpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \sendpipeline
+ROLLBACK \bind \sendpipeline
+BEGIN \bind \sendpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \sendpipeline
+COMMIT \bind \sendpipeline
\endpipeline
-- COPY FROM STDIN
\startpipeline
-SELECT $1 \bind 'val1' \g
-COPY psql_pipeline FROM STDIN \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+COPY psql_pipeline FROM STDIN \bind \sendpipeline
\endpipeline
2 test2
\.
-- COPY FROM STDIN with \flushrequest + \getresults
\startpipeline
-SELECT $1 \bind 'val1' \g
-COPY psql_pipeline FROM STDIN \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+COPY psql_pipeline FROM STDIN \bind \sendpipeline
\flushrequest
\getresults
3 test3
@@ -85,8 +85,8 @@ COPY psql_pipeline FROM STDIN \bind \g
-- COPY FROM STDIN with \syncpipeline + \getresults
\startpipeline
-SELECT $1 \bind 'val1' \g
-COPY psql_pipeline FROM STDIN \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+COPY psql_pipeline FROM STDIN \bind \sendpipeline
\syncpipeline
\getresults
4 test4
@@ -95,22 +95,22 @@ COPY psql_pipeline FROM STDIN \bind \g
-- COPY TO STDOUT
\startpipeline
-SELECT $1 \bind 'val1' \g
-copy psql_pipeline TO STDOUT \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+copy psql_pipeline TO STDOUT \bind \sendpipeline
\endpipeline
-- COPY TO STDOUT with \flushrequest + \getresults
\startpipeline
-SELECT $1 \bind 'val1' \g
-copy psql_pipeline TO STDOUT \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+copy psql_pipeline TO STDOUT \bind \sendpipeline
\flushrequest
\getresults
\endpipeline
-- COPY TO STDOUT with \syncpipeline + \getresults
\startpipeline
-SELECT $1 \bind 'val1' \g
-copy psql_pipeline TO STDOUT \bind \g
+SELECT $1 \bind 'val1' \sendpipeline
+copy psql_pipeline TO STDOUT \bind \sendpipeline
\syncpipeline
\getresults
\endpipeline
@@ -120,22 +120,22 @@ copy psql_pipeline TO STDOUT \bind \g
SELECT $1 \parse ''
SELECT $1, $2 \parse ''
SELECT $2 \parse pipeline_1
-\bind_named '' 1 2 \g
-\bind_named pipeline_1 2 \g
+\bind_named '' 1 2 \sendpipeline
+\bind_named pipeline_1 2 \sendpipeline
\endpipeline
-- \getresults displays all results preceding a \flushrequest.
\startpipeline
-SELECT $1 \bind 1 \g
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 1 \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\flushrequest
\getresults
\endpipeline
-- \getresults displays all results preceding a \syncpipeline.
\startpipeline
-SELECT $1 \bind 1 \g
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 1 \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\syncpipeline
\getresults
\endpipeline
@@ -143,7 +143,7 @@ SELECT $1 \bind 2 \g
-- \getresults immediately returns if there is no result to fetch.
\startpipeline
\getresults
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\getresults
\flushrequest
\endpipeline
@@ -151,42 +151,42 @@ SELECT $1 \bind 2 \g
-- \getresults only fetches results preceding a \flushrequest.
\startpipeline
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\flushrequest
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\getresults
\endpipeline
-- \getresults only fetches results preceding a \syncpipeline.
\startpipeline
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\syncpipeline
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\getresults
\endpipeline
-- Use pipeline with chunked results for both \getresults and \endpipeline.
\startpipeline
\set FETCH_COUNT 10
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\flushrequest
\getresults
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
\unset FETCH_COUNT
-- \getresults with specific number of requested results.
\startpipeline
-SELECT $1 \bind 1 \g
-SELECT $1 \bind 2 \g
-SELECT $1 \bind 3 \g
+SELECT $1 \bind 1 \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
+SELECT $1 \bind 3 \sendpipeline
\echo :PIPELINE_SYNC_COUNT
\syncpipeline
\echo :PIPELINE_SYNC_COUNT
\echo :PIPELINE_RESULT_COUNT
\getresults 1
\echo :PIPELINE_RESULT_COUNT
-SELECT $1 \bind 4 \g
+SELECT $1 \bind 4 \sendpipeline
\getresults 3
\echo :PIPELINE_RESULT_COUNT
\endpipeline
@@ -195,7 +195,7 @@ SELECT $1 \bind 4 \g
\startpipeline
\syncpipeline
\syncpipeline
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
\flushrequest
\getresults 2
\getresults 1
@@ -203,9 +203,9 @@ SELECT $1 \bind 1 \g
-- \getresults 0 should get all the results.
\startpipeline
-SELECT $1 \bind 1 \g
-SELECT $1 \bind 2 \g
-SELECT $1 \bind 3 \g
+SELECT $1 \bind 1 \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
+SELECT $1 \bind 3 \sendpipeline
\syncpipeline
\getresults 0
\endpipeline
@@ -221,36 +221,36 @@ SELECT $1 \bind 3 \g
-- pipeline usable.
\startpipeline
SELECT 1;
-SELECT $1 \bind 'val1' \g
+SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
-- After an aborted pipeline, commands after a \syncpipeline should be
-- displayed.
\startpipeline
-SELECT $1 \bind \g
+SELECT $1 \bind \sendpipeline
\syncpipeline
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
\endpipeline
-- For an incorrect number of parameters, the pipeline is aborted and
-- the following queries will not be executed.
\startpipeline
-SELECT \bind 'val1' \g
-SELECT $1 \bind 'val1' \g
+SELECT \bind 'val1' \sendpipeline
+SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
-- An explicit transaction with an error needs to be rollbacked after
-- the pipeline.
\startpipeline
-BEGIN \bind \g
-INSERT INTO psql_pipeline VALUES ($1) \bind 1 \g
-ROLLBACK \bind \g
+BEGIN \bind \sendpipeline
+INSERT INTO psql_pipeline VALUES ($1) \bind 1 \sendpipeline
+ROLLBACK \bind \sendpipeline
\endpipeline
ROLLBACK;
-- \watch sends a simple query, something not allowed within a pipeline.
\startpipeline
-SELECT \bind \g
+SELECT \bind \sendpipeline
\watch 1
\endpipeline
@@ -258,7 +258,7 @@ SELECT \bind \g
-- and the pipeline should still be usable.
\startpipeline
SELECT $1 \bind 1 \gdesc
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
\endpipeline
-- \gset is not allowed in a pipeline, pipeline should still be usable.
@@ -267,54 +267,72 @@ SELECT $1 as i, $2 as j \parse ''
SELECT $1 as k, $2 as l \parse 'second'
\bind_named '' 1 2 \gset
\bind_named second 1 2 \gset pref02_ \echo :pref02_i :pref02_j
-\bind_named '' 1 2 \g
+\bind_named '' 1 2 \sendpipeline
\endpipeline
--- \gx is not allowed, pipeline should still be usable.
+-- \g and \gx are not allowed, pipeline should still be usable.
\startpipeline
+SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \g filename
+SELECT $1 \bind 1 \g |cat
+SELECT $1 \bind 1 \g (format=unaligned tuples_only=on)
SELECT $1 \bind 1 \gx
+SELECT $1 \bind 1 \gx filename
+SELECT $1 \bind 1 \gx |cat
+SELECT $1 \bind 1 \gx (format=unaligned tuples_only=on)
\reset
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
\endpipeline
-- \gx warning should be emitted in an aborted pipeline, with
-- pipeline still usable.
\startpipeline
-SELECT $1 \bind \g
+SELECT $1 \bind \sendpipeline
\flushrequest
\getresults
SELECT $1 \bind 1 \gx
\endpipeline
+-- \sendpipeline is not allowed outside of a pipeline
+\sendpipeline
+SELECT $1 \bind 1 \sendpipeline
+\reset
+
+-- \sendpipeline is not allowed if not preceded by \bind or \bind_named
+\startpipeline
+\sendpipeline
+SELECT 1 \sendpipeline
+\endpipeline
+
-- \gexec is not allowed, pipeline should still be usable.
\startpipeline
SELECT 'INSERT INTO psql_pipeline(a) SELECT generate_series(1, 10)' \parse 'insert_stmt'
\bind_named insert_stmt \gexec
-\bind_named insert_stmt \g
+\bind_named insert_stmt \sendpipeline
SELECT COUNT(*) FROM psql_pipeline \bind \g
\endpipeline
-- After an error, pipeline is aborted and requires \syncpipeline to be
-- reusable.
\startpipeline
-SELECT $1 \bind \g
-SELECT $1 \bind 1 \g
+SELECT $1 \bind \sendpipeline
+SELECT $1 \bind 1 \sendpipeline
SELECT $1 \parse a
-\bind_named a 1 \g
+\bind_named a 1 \sendpipeline
\close a
\flushrequest
\getresults
-- Pipeline is aborted.
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
SELECT $1 \parse a
-\bind_named a 1 \g
+\bind_named a 1 \sendpipeline
\close a
-- Sync allows pipeline to recover.
\syncpipeline
\getresults
-SELECT $1 \bind 1 \g
+SELECT $1 \bind 1 \sendpipeline
SELECT $1 \parse a
-\bind_named a 1 \g
+\bind_named a 1 \sendpipeline
\close a
\flushrequest
\getresults
@@ -322,10 +340,10 @@ SELECT $1 \parse a
-- In an aborted pipeline, \getresults 1 aborts commands one at a time.
\startpipeline
-SELECT $1 \bind \g
-SELECT $1 \bind 1 \g
+SELECT $1 \bind \sendpipeline
+SELECT $1 \bind 1 \sendpipeline
SELECT $1 \parse a
-\bind_named a 1 \g
+\bind_named a 1 \sendpipeline
\syncpipeline
\getresults 1
\getresults 1
@@ -337,10 +355,10 @@ SELECT $1 \parse a
-- Test chunked results with an aborted pipeline.
\startpipeline
\set FETCH_COUNT 10
-SELECT $1 \bind \g
+SELECT $1 \bind \sendpipeline
\flushrequest
\getresults
-SELECT $1 \bind \g
+SELECT $1 \bind \sendpipeline
\endpipeline
\unset FETCH_COUNT
@@ -356,7 +374,7 @@ select 1;
-- Error messages accumulate and are repeated.
\startpipeline
-SELECT 1 \bind \g
+SELECT 1 \bind \sendpipeline
SELECT 1;
SELECT 1;
\endpipeline
@@ -371,66 +389,66 @@ SELECT 1;
-- commit the implicit transaction block. The first command after a
-- sync will not be seen as belonging to a pipeline.
\startpipeline
-SET LOCAL statement_timeout='1h' \bind \g
-SHOW statement_timeout \bind \g
+SET LOCAL statement_timeout='1h' \bind \sendpipeline
+SHOW statement_timeout \bind \sendpipeline
\syncpipeline
-SHOW statement_timeout \bind \g
-SET LOCAL statement_timeout='2h' \bind \g
-SHOW statement_timeout \bind \g
+SHOW statement_timeout \bind \sendpipeline
+SET LOCAL statement_timeout='2h' \bind \sendpipeline
+SHOW statement_timeout \bind \sendpipeline
\endpipeline
-- REINDEX CONCURRENTLY fails if not the first command in a pipeline.
\startpipeline
-SELECT $1 \bind 1 \g
-REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 1 \sendpipeline
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
-- REINDEX CONCURRENTLY works if it is the first command in a pipeline.
\startpipeline
-REINDEX TABLE CONCURRENTLY psql_pipeline \bind \g
-SELECT $1 \bind 2 \g
+REINDEX TABLE CONCURRENTLY psql_pipeline \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
-- Subtransactions are not allowed in a pipeline.
\startpipeline
-SAVEPOINT a \bind \g
-SELECT $1 \bind 1 \g
-ROLLBACK TO SAVEPOINT a \bind \g
-SELECT $1 \bind 2 \g
+SAVEPOINT a \bind \sendpipeline
+SELECT $1 \bind 1 \sendpipeline
+ROLLBACK TO SAVEPOINT a \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
-- LOCK fails as the first command in a pipeline, as not seen in an
-- implicit transaction block.
\startpipeline
-LOCK psql_pipeline \bind \g
-SELECT $1 \bind 2 \g
+LOCK psql_pipeline \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
-- LOCK succeeds as it is not the first command in a pipeline,
-- seen in an implicit transaction block.
\startpipeline
-SELECT $1 \bind 1 \g
-LOCK psql_pipeline \bind \g
-SELECT $1 \bind 2 \g
+SELECT $1 \bind 1 \sendpipeline
+LOCK psql_pipeline \bind \sendpipeline
+SELECT $1 \bind 2 \sendpipeline
\endpipeline
-- VACUUM works as the first command in a pipeline.
\startpipeline
-VACUUM psql_pipeline \bind \g
+VACUUM psql_pipeline \bind \sendpipeline
\endpipeline
-- VACUUM fails when not the first command in a pipeline.
\startpipeline
-SELECT 1 \bind \g
-VACUUM psql_pipeline \bind \g
+SELECT 1 \bind \sendpipeline
+VACUUM psql_pipeline \bind \sendpipeline
\endpipeline
-- VACUUM works after a \syncpipeline.
\startpipeline
-SELECT 1 \bind \g
+SELECT 1 \bind \sendpipeline
\syncpipeline
-VACUUM psql_pipeline \bind \g
+VACUUM psql_pipeline \bind \sendpipeline
\endpipeline
-- Clean up
--
2.39.5 (Apple Git-154)
[application/octet-stream] v02-0002-psql-Allow-to-add-queries-in-an-ongoing-pipeline.patch (7.5K, ../../CAO6_Xqq4JEHRaGyC=r+dpi0Ejbp2P3aUo-KfR2NuipCvE-o_ZA@mail.gmail.com/3-v02-0002-psql-Allow-to-add-queries-in-an-ongoing-pipeline.patch)
download | inline diff:
From 2ebe091c9fcc5cd35bbbc02ece90645539330ebd Mon Sep 17 00:00:00 2001
From: Anthonin Bonnefoy <[email protected]>
Date: Wed, 5 Mar 2025 14:55:33 +0100
Subject: psql: Allow ';' to add queries in an ongoing pipeline
Currently, the only way to pipe queries in an ongoing pipeline is to
leverage psql meta-commands to create extended queries such as \bind,
\parse or \bind_named. This prevents using psql's pipeline on existing
scripts as it would require to convert all queries to use those
meta-commands.
This patch modifies ';' behaviour within an active pipeline and send all
queries as extended queries, allowing them to be piped in a pipeline.
---
doc/src/sgml/ref/psql-ref.sgml | 21 +++++----
src/bin/psql/command.c | 7 +++
src/bin/psql/common.c | 10 +++-
src/test/regress/expected/psql_pipeline.out | 52 +++++++++++++--------
src/test/regress/sql/psql_pipeline.sql | 24 ++++++----
5 files changed, 77 insertions(+), 37 deletions(-)
diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml
index cddf6e07531..2763486e268 100644
--- a/doc/src/sgml/ref/psql-ref.sgml
+++ b/doc/src/sgml/ref/psql-ref.sgml
@@ -3698,14 +3698,19 @@ testdb=> <userinput>\setenv LESS -imx4F</userinput>
</para>
<para>
- Pipeline mode requires the use of the extended query protocol. All
- queries need to be sent using the meta-commands
- <literal>\bind</literal>, <literal>\bind_named</literal>,
- <literal>\close</literal> or <literal>\parse</literal>. While a
- pipeline is ongoing, <literal>\sendpipeline</literal> will append the
- current query buffer to the pipeline. Other meta-commands like
- <literal>\g</literal>, <literal>\gx</literal> or <literal>\gdesc</literal>
- are not allowed in pipeline mode.
+ Pipeline mode requires the use of the extended query protocol. Queries
+ can be sent using the meta-commands <literal>\bind</literal>,
+ <literal>\bind_named</literal>, <literal>\close</literal> or
+ <literal>\parse</literal>. While a pipeline is ongoing,
+ <literal>\sendpipeline</literal> will append the current query
+ buffer to the pipeline. Other meta-commands like <literal>\g</literal>,
+ <literal>\gx</literal> or <literal>\gdesc</literal> are not allowed
+ in pipeline mode.
+ </para>
+
+ <para>
+ Queries can also be sent using <literal>;</literal>. While a pipeline is
+ ongoing, they will automatically be sent using extended query protocol.
</para>
<para>
diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c
index 7670c20b29e..3e7eb086367 100644
--- a/src/bin/psql/command.c
+++ b/src/bin/psql/command.c
@@ -3282,6 +3282,13 @@ exec_command_watch(PsqlScanState scan_state, bool active_branch,
int iter = 0;
int min_rows = 0;
+ if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pg_log_error("\\watch not allowed in pipeline mode");
+ clean_extended_state();
+ success = false;
+ }
+
/*
* Parse arguments. We allow either an unlabeled interval or
* "name=value", where name is from the set ('i', 'interval', 'c',
diff --git a/src/bin/psql/common.c b/src/bin/psql/common.c
index ed340a466f9..5249336bcf2 100644
--- a/src/bin/psql/common.c
+++ b/src/bin/psql/common.c
@@ -1668,7 +1668,15 @@ ExecQueryAndProcessResults(const char *query,
}
break;
case PSQL_SEND_QUERY:
- success = PQsendQuery(pset.db, query);
+ if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ success = PQsendQueryParams(pset.db, query,
+ 0, NULL, NULL, NULL, NULL, 0);
+ if (success)
+ pset.piped_commands++;
+ }
+ else
+ success = PQsendQuery(pset.db, query);
break;
}
diff --git a/src/test/regress/expected/psql_pipeline.out b/src/test/regress/expected/psql_pipeline.out
index 53f2edfd986..f05429b36ac 100644
--- a/src/test/regress/expected/psql_pipeline.out
+++ b/src/test/regress/expected/psql_pipeline.out
@@ -387,24 +387,33 @@ SELECT $1 \bind 3 \sendpipeline
(1 row)
\endpipeline
+-- Queries can be pipelined ';'. They will be sent using extended protocol.
+\startpipeline
+SELECT 1;
+SELECT $1 \bind 'val1' \sendpipeline
+SELECT 2;
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ val1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
--
-- Pipeline errors
--
-- \endpipeline outside of pipeline should fail
\endpipeline
cannot send pipeline when not in pipeline mode
--- Query using simple protocol should not be sent and should leave the
--- pipeline usable.
-\startpipeline
-SELECT 1;
-PQsendQuery not allowed in pipeline mode
-SELECT $1 \bind 'val1' \sendpipeline
-\endpipeline
- ?column?
-----------
- val1
-(1 row)
-
-- After an aborted pipeline, commands after a \syncpipeline should be
-- displayed.
\startpipeline
@@ -425,6 +434,12 @@ SELECT \bind 'val1' \sendpipeline
SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
ERROR: bind message supplies 1 parameters, but prepared statement "" requires 0
+-- Using ';' with a parameter will trigger an incorrect parameter errors.
+\startpipeline
+SELECT $1;
+SELECT 1;
+\endpipeline
+ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
-- An explicit transaction with an error needs to be rollbacked after
-- the pipeline.
\startpipeline
@@ -439,8 +454,7 @@ ROLLBACK;
\startpipeline
SELECT \bind \sendpipeline
\watch 1
-PQsendQuery not allowed in pipeline mode
-
+\watch not allowed in pipeline mode
\endpipeline
--
(1 row)
@@ -619,11 +633,11 @@ select 1;
-- Error messages accumulate and are repeated.
\startpipeline
SELECT 1 \bind \sendpipeline
-SELECT 1;
-PQsendQuery not allowed in pipeline mode
-SELECT 1;
-PQsendQuery not allowed in pipeline mode
-PQsendQuery not allowed in pipeline mode
+\gdesc
+synchronous command execution functions are not allowed in pipeline mode
+\gdesc
+synchronous command execution functions are not allowed in pipeline mode
+synchronous command execution functions are not allowed in pipeline mode
\endpipeline
?column?
----------
diff --git a/src/test/regress/sql/psql_pipeline.sql b/src/test/regress/sql/psql_pipeline.sql
index 256081dfd5f..76c0ece01af 100644
--- a/src/test/regress/sql/psql_pipeline.sql
+++ b/src/test/regress/sql/psql_pipeline.sql
@@ -210,6 +210,13 @@ SELECT $1 \bind 3 \sendpipeline
\getresults 0
\endpipeline
+-- Queries can be pipelined ';'. They will be sent using extended protocol.
+\startpipeline
+SELECT 1;
+SELECT $1 \bind 'val1' \sendpipeline
+SELECT 2;
+\endpipeline
+
--
-- Pipeline errors
--
@@ -217,13 +224,6 @@ SELECT $1 \bind 3 \sendpipeline
-- \endpipeline outside of pipeline should fail
\endpipeline
--- Query using simple protocol should not be sent and should leave the
--- pipeline usable.
-\startpipeline
-SELECT 1;
-SELECT $1 \bind 'val1' \sendpipeline
-\endpipeline
-
-- After an aborted pipeline, commands after a \syncpipeline should be
-- displayed.
\startpipeline
@@ -239,6 +239,12 @@ SELECT \bind 'val1' \sendpipeline
SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
+-- Using ';' with a parameter will trigger an incorrect parameter errors.
+\startpipeline
+SELECT $1;
+SELECT 1;
+\endpipeline
+
-- An explicit transaction with an error needs to be rollbacked after
-- the pipeline.
\startpipeline
@@ -375,8 +381,8 @@ select 1;
-- Error messages accumulate and are repeated.
\startpipeline
SELECT 1 \bind \sendpipeline
-SELECT 1;
-SELECT 1;
+\gdesc
+\gdesc
\endpipeline
--
--
2.39.5 (Apple Git-154)
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-17 10:19 Michael Paquier <[email protected]>
parent: Anthonin Bonnefoy <[email protected]>
2 siblings, 0 replies; 41+ messages in thread
From: Michael Paquier @ 2025-03-17 10:19 UTC (permalink / raw)
To: Anthonin Bonnefoy <[email protected]>; +Cc: Daniel Verite <[email protected]>; Jelte Fennema-Nio <[email protected]>; pgsql-hackers
On Mon, Mar 17, 2025 at 10:50:50AM +0100, Anthonin Bonnefoy wrote:
> 0001: This introduces the \sendpipeline meta-command and forbid \g in
> a pipeline. This is to fix the formatting options of \g that are not
> supported in a pipeline.
>
> 0002: Allows ';' to send a query using extended protocol when within a
> pipeline by using PQsendQueryParams with 0 parameters. It is not
> possible to send parameters with extended protocol this way and
> everything will be propagated through the query string, similar to a
> simple query.
Thanks for sending a new patch set. I was planning to look at the
situation tomorrow, and you have beaten me to it.
The split makes sense, and I'm OK with 0001. 0002 is going to require
a much closer lookup.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-18 00:50 Michael Paquier <[email protected]>
parent: Anthonin Bonnefoy <[email protected]>
2 siblings, 1 reply; 41+ messages in thread
From: Michael Paquier @ 2025-03-18 00:50 UTC (permalink / raw)
To: Anthonin Bonnefoy <[email protected]>; +Cc: Daniel Verite <[email protected]>; Jelte Fennema-Nio <[email protected]>; pgsql-hackers
On Mon, Mar 17, 2025 at 10:50:50AM +0100, Anthonin Bonnefoy wrote:
> 0001: This introduces the \sendpipeline meta-command and forbid \g in
> a pipeline. This is to fix the formatting options of \g that are not
> supported in a pipeline.
- count
--------
- 4
-(1 row)
This removal done in the regression tests was not intentional.
I have done some reordering of the code around the new meta-command so
as things are ordered alphabetically, and applied the result.
> 0002: Allows ';' to send a query using extended protocol when within a
> pipeline by using PQsendQueryParams with 0 parameters. It is not
> possible to send parameters with extended protocol this way and
> everything will be propagated through the query string, similar to a
> simple query.
I like the simplicity of what you are doing here, relying on
PSQL_SEND_QUERY being the default so as we use PQsendQueryParams()
with no parameters rather than PQsendQuery() when the pipeline mode is
not off.
How about adding a check on PIPELINE_COMMAND_COUNT when sending a
query through this path? Should we check for more scenarios with
syncs and flushes as well when sending these queries?
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-18 08:55 Anthonin Bonnefoy <[email protected]>
parent: Michael Paquier <[email protected]>
0 siblings, 2 replies; 41+ messages in thread
From: Anthonin Bonnefoy @ 2025-03-18 08:55 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Daniel Verite <[email protected]>; Jelte Fennema-Nio <[email protected]>; pgsql-hackers
On Tue, Mar 18, 2025 at 1:50 AM Michael Paquier <[email protected]> wrote:
> - count
> --------
> - 4
> -(1 row)
>
> This removal done in the regression tests was not intentional.
Yes, thanks for fixing that.
> How about adding a check on PIPELINE_COMMAND_COUNT when sending a
> query through this path? Should we check for more scenarios with
> syncs and flushes as well when sending these queries?
I've added additional tests when piping queries with ';':
- I've reused the same scenario with \sendpipeline: single query,
multiple queries, flushes, syncs, using COPY...
- Using ';' will replace the unnamed prepared statement. It's a bit
different from expected as a simple query will delete the unnamed
prepared statement.
- Sending an extended query prepared with \bind using a ';' on a
newline, though this is not specific to pipelining. The scanned
semicolon triggers the call to SendQuery, processing the buffered
extended query. It's a bit unusual but that's the current behaviour.
Attachments:
[application/octet-stream] v03-0001-psql-Allow-to-add-queries-in-an-ongoing-pipeline.patch (13.5K, ../../CAO6_Xqo=fNh0KCRZG7T18OZfjiabeL-t4C6mkXsz-3dAZ1YwTA@mail.gmail.com/2-v03-0001-psql-Allow-to-add-queries-in-an-ongoing-pipeline.patch)
download | inline diff:
From ae9ce7e44c1e2502429920c5d305e1ffcd30b1a5 Mon Sep 17 00:00:00 2001
From: Anthonin Bonnefoy <[email protected]>
Date: Wed, 5 Mar 2025 14:55:33 +0100
Subject: psql: Allow ';' to add queries in an ongoing pipeline
Currently, the only way to pipe queries in an ongoing pipeline is to
leverage psql meta-commands to create extended queries such as \bind,
\parse or \bind_named. This prevents using psql's pipeline on existing
scripts as it would require to convert all queries to use those
meta-commands.
This patch modifies ';' behaviour within an active pipeline and send all
queries as extended queries, allowing them to be piped in a pipeline.
---
doc/src/sgml/ref/psql-ref.sgml | 21 +-
src/bin/psql/command.c | 7 +
src/bin/psql/common.c | 10 +-
src/test/regress/expected/psql_pipeline.out | 292 ++++++++++++++++++--
src/test/regress/sql/psql_pipeline.sql | 139 +++++++++-
5 files changed, 429 insertions(+), 40 deletions(-)
diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml
index cddf6e07531..2763486e268 100644
--- a/doc/src/sgml/ref/psql-ref.sgml
+++ b/doc/src/sgml/ref/psql-ref.sgml
@@ -3698,14 +3698,19 @@ testdb=> <userinput>\setenv LESS -imx4F</userinput>
</para>
<para>
- Pipeline mode requires the use of the extended query protocol. All
- queries need to be sent using the meta-commands
- <literal>\bind</literal>, <literal>\bind_named</literal>,
- <literal>\close</literal> or <literal>\parse</literal>. While a
- pipeline is ongoing, <literal>\sendpipeline</literal> will append the
- current query buffer to the pipeline. Other meta-commands like
- <literal>\g</literal>, <literal>\gx</literal> or <literal>\gdesc</literal>
- are not allowed in pipeline mode.
+ Pipeline mode requires the use of the extended query protocol. Queries
+ can be sent using the meta-commands <literal>\bind</literal>,
+ <literal>\bind_named</literal>, <literal>\close</literal> or
+ <literal>\parse</literal>. While a pipeline is ongoing,
+ <literal>\sendpipeline</literal> will append the current query
+ buffer to the pipeline. Other meta-commands like <literal>\g</literal>,
+ <literal>\gx</literal> or <literal>\gdesc</literal> are not allowed
+ in pipeline mode.
+ </para>
+
+ <para>
+ Queries can also be sent using <literal>;</literal>. While a pipeline is
+ ongoing, they will automatically be sent using extended query protocol.
</para>
<para>
diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c
index a87ff7e4597..bbe337780ff 100644
--- a/src/bin/psql/command.c
+++ b/src/bin/psql/command.c
@@ -3282,6 +3282,13 @@ exec_command_watch(PsqlScanState scan_state, bool active_branch,
int iter = 0;
int min_rows = 0;
+ if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ pg_log_error("\\watch not allowed in pipeline mode");
+ clean_extended_state();
+ success = false;
+ }
+
/*
* Parse arguments. We allow either an unlabeled interval or
* "name=value", where name is from the set ('i', 'interval', 'c',
diff --git a/src/bin/psql/common.c b/src/bin/psql/common.c
index ed340a466f9..5249336bcf2 100644
--- a/src/bin/psql/common.c
+++ b/src/bin/psql/common.c
@@ -1668,7 +1668,15 @@ ExecQueryAndProcessResults(const char *query,
}
break;
case PSQL_SEND_QUERY:
- success = PQsendQuery(pset.db, query);
+ if (PQpipelineStatus(pset.db) != PQ_PIPELINE_OFF)
+ {
+ success = PQsendQueryParams(pset.db, query,
+ 0, NULL, NULL, NULL, NULL, 0);
+ if (success)
+ pset.piped_commands++;
+ }
+ else
+ success = PQsendQuery(pset.db, query);
break;
}
diff --git a/src/test/regress/expected/psql_pipeline.out b/src/test/regress/expected/psql_pipeline.out
index 68e3c19ea05..7dddf26a9fd 100644
--- a/src/test/regress/expected/psql_pipeline.out
+++ b/src/test/regress/expected/psql_pipeline.out
@@ -386,6 +386,262 @@ SELECT $1 \bind 3 \sendpipeline
3
(1 row)
+\endpipeline
+--
+-- Tests pipelining queries with ';'
+--
+-- Single query sent with ';'
+\startpipeline
+SELECT 1;
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+-- Multiple queries sent with ';'
+\startpipeline
+SELECT 1;
+SELECT 2;
+SELECT 3;
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+ ?column?
+----------
+ 3
+(1 row)
+
+-- Multiple queries on the same line can be piped with ';'
+\startpipeline
+SELECT 1; SELECT 2; SELECT 3
+;
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+ ?column?
+----------
+ 3
+(1 row)
+
+-- Test \flush with queries piped with ';'
+\startpipeline
+\flush
+SELECT 1;
+\flush
+SELECT 2;
+SELECT 3;
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+ ?column?
+----------
+ 3
+(1 row)
+
+-- Send multiple syncs with queries piped with ';'
+\startpipeline
+\echo :PIPELINE_COMMAND_COUNT
+0
+\echo :PIPELINE_SYNC_COUNT
+0
+\echo :PIPELINE_RESULT_COUNT
+0
+SELECT 1;
+\syncpipeline
+\syncpipeline
+SELECT 2;
+\syncpipeline
+SELECT 3;
+\echo :PIPELINE_COMMAND_COUNT
+1
+\echo :PIPELINE_SYNC_COUNT
+3
+\echo :PIPELINE_RESULT_COUNT
+2
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+ ?column?
+----------
+ 3
+(1 row)
+
+-- Mix queries piped with ';' and \sendpipeline
+\startpipeline
+SELECT 1;
+SELECT $1 \bind 'val1' \sendpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
+SELECT 2;
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ val1
+(1 row)
+
+ ?column? | ?column?
+----------+----------
+ val2 | val3
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+-- Piping a query with ';' will replace the unnamed prepared statement
+\startpipeline
+SELECT $1 \parse ''
+SELECT 1;
+\bind_named ''
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+-- An extended query can be piped by a ';' after a newline
+\startpipeline
+SELECT $1 \bind 1
+;
+SELECT 2;
+\endpipeline
+ ?column?
+----------
+ 1
+(1 row)
+
+ ?column?
+----------
+ 2
+(1 row)
+
+-- COPY FROM STDIN, using ';'
+\startpipeline
+SELECT 'val1';
+COPY psql_pipeline FROM STDIN;
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+-- COPY FROM STDIN with \flushrequest + \getresults, using ';'
+\startpipeline
+SELECT 'val1';
+COPY psql_pipeline FROM STDIN;
+\flushrequest
+\getresults
+ ?column?
+----------
+ val1
+(1 row)
+
+message type 0x5a arrived from server while idle
+\endpipeline
+-- COPY FROM STDIN with \syncpipeline + \getresults, using ';'
+\startpipeline
+SELECT 'val1';
+COPY psql_pipeline FROM STDIN;
+\syncpipeline
+\getresults
+ ?column?
+----------
+ val1
+(1 row)
+
+\endpipeline
+-- COPY TO STDOUT, using ';'
+\startpipeline
+SELECT 'val1';
+copy psql_pipeline TO STDOUT;
+\endpipeline
+ ?column?
+----------
+ val1
+(1 row)
+
+1 \N
+2 test2
+3 test3
+4 test4
+20 test2
+30 test3
+40 test4
+-- COPY TO STDOUT with \flushrequest + \getresults, using ';'
+\startpipeline
+SELECT 'val1';
+copy psql_pipeline TO STDOUT;
+\flushrequest
+\getresults
+ ?column?
+----------
+ val1
+(1 row)
+
+1 \N
+2 test2
+3 test3
+4 test4
+20 test2
+30 test3
+40 test4
+\endpipeline
+-- COPY TO STDOUT with \syncpipeline + \getresults, using ';'
+\startpipeline
+SELECT 'val1';
+copy psql_pipeline TO STDOUT;
+\syncpipeline
+\getresults
+ ?column?
+----------
+ val1
+(1 row)
+
+1 \N
+2 test2
+3 test3
+4 test4
+20 test2
+30 test3
+40 test4
\endpipeline
--
-- Pipeline errors
@@ -393,18 +649,6 @@ SELECT $1 \bind 3 \sendpipeline
-- \endpipeline outside of pipeline should fail
\endpipeline
cannot send pipeline when not in pipeline mode
--- Query using simple protocol should not be sent and should leave the
--- pipeline usable.
-\startpipeline
-SELECT 1;
-PQsendQuery not allowed in pipeline mode
-SELECT $1 \bind 'val1' \sendpipeline
-\endpipeline
- ?column?
-----------
- val1
-(1 row)
-
-- After an aborted pipeline, commands after a \syncpipeline should be
-- displayed.
\startpipeline
@@ -425,6 +669,13 @@ SELECT \bind 'val1' \sendpipeline
SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
ERROR: bind message supplies 1 parameters, but prepared statement "" requires 0
+-- Using ';' with a parameter will trigger an incorrect parameter error and
+-- abort the pipeline
+\startpipeline
+SELECT $1;
+SELECT 1;
+\endpipeline
+ERROR: bind message supplies 0 parameters, but prepared statement "" requires 1
-- An explicit transaction with an error needs to be rollbacked after
-- the pipeline.
\startpipeline
@@ -435,12 +686,11 @@ ROLLBACK \bind \sendpipeline
ERROR: duplicate key value violates unique constraint "psql_pipeline_pkey"
DETAIL: Key (a)=(1) already exists.
ROLLBACK;
--- \watch sends a simple query, something not allowed within a pipeline.
+-- \watch is not allowed within a pipeline.
\startpipeline
SELECT \bind \sendpipeline
\watch 1
-PQsendQuery not allowed in pipeline mode
-
+\watch not allowed in pipeline mode
\endpipeline
--
(1 row)
@@ -530,7 +780,7 @@ SELECT COUNT(*) FROM psql_pipeline \bind \sendpipeline
count
-------
- 4
+ 7
(1 row)
-- After an error, pipeline is aborted and requires \syncpipeline to be
@@ -617,11 +867,11 @@ select 1;
-- Error messages accumulate and are repeated.
\startpipeline
SELECT 1 \bind \sendpipeline
-SELECT 1;
-PQsendQuery not allowed in pipeline mode
-SELECT 1;
-PQsendQuery not allowed in pipeline mode
-PQsendQuery not allowed in pipeline mode
+\gdesc
+synchronous command execution functions are not allowed in pipeline mode
+\gdesc
+synchronous command execution functions are not allowed in pipeline mode
+synchronous command execution functions are not allowed in pipeline mode
\endpipeline
?column?
----------
diff --git a/src/test/regress/sql/psql_pipeline.sql b/src/test/regress/sql/psql_pipeline.sql
index e4d7e614af3..6f76adcba82 100644
--- a/src/test/regress/sql/psql_pipeline.sql
+++ b/src/test/regress/sql/psql_pipeline.sql
@@ -210,6 +210,125 @@ SELECT $1 \bind 3 \sendpipeline
\getresults 0
\endpipeline
+--
+-- Tests pipelining queries with ';'
+--
+
+-- Single query sent with ';'
+\startpipeline
+SELECT 1;
+\endpipeline
+
+-- Multiple queries sent with ';'
+\startpipeline
+SELECT 1;
+SELECT 2;
+SELECT 3;
+\endpipeline
+
+-- Multiple queries on the same line can be piped with ';'
+\startpipeline
+SELECT 1; SELECT 2; SELECT 3
+;
+\endpipeline
+
+-- Test \flush with queries piped with ';'
+\startpipeline
+\flush
+SELECT 1;
+\flush
+SELECT 2;
+SELECT 3;
+\endpipeline
+
+-- Send multiple syncs with queries piped with ';'
+\startpipeline
+\echo :PIPELINE_COMMAND_COUNT
+\echo :PIPELINE_SYNC_COUNT
+\echo :PIPELINE_RESULT_COUNT
+SELECT 1;
+\syncpipeline
+\syncpipeline
+SELECT 2;
+\syncpipeline
+SELECT 3;
+\echo :PIPELINE_COMMAND_COUNT
+\echo :PIPELINE_SYNC_COUNT
+\echo :PIPELINE_RESULT_COUNT
+\endpipeline
+
+-- Mix queries piped with ';' and \sendpipeline
+\startpipeline
+SELECT 1;
+SELECT $1 \bind 'val1' \sendpipeline
+SELECT $1, $2 \bind 'val2' 'val3' \sendpipeline
+SELECT 2;
+\endpipeline
+
+-- Piping a query with ';' will replace the unnamed prepared statement
+\startpipeline
+SELECT $1 \parse ''
+SELECT 1;
+\bind_named ''
+\endpipeline
+
+-- An extended query can be piped by a ';' after a newline
+\startpipeline
+SELECT $1 \bind 1
+;
+SELECT 2;
+\endpipeline
+
+-- COPY FROM STDIN, using ';'
+\startpipeline
+SELECT 'val1';
+COPY psql_pipeline FROM STDIN;
+\endpipeline
+20 test2
+\.
+
+-- COPY FROM STDIN with \flushrequest + \getresults, using ';'
+\startpipeline
+SELECT 'val1';
+COPY psql_pipeline FROM STDIN;
+\flushrequest
+\getresults
+30 test3
+\.
+\endpipeline
+
+-- COPY FROM STDIN with \syncpipeline + \getresults, using ';'
+\startpipeline
+SELECT 'val1';
+COPY psql_pipeline FROM STDIN;
+\syncpipeline
+\getresults
+40 test4
+\.
+\endpipeline
+
+-- COPY TO STDOUT, using ';'
+\startpipeline
+SELECT 'val1';
+copy psql_pipeline TO STDOUT;
+\endpipeline
+
+-- COPY TO STDOUT with \flushrequest + \getresults, using ';'
+\startpipeline
+SELECT 'val1';
+copy psql_pipeline TO STDOUT;
+\flushrequest
+\getresults
+\endpipeline
+
+-- COPY TO STDOUT with \syncpipeline + \getresults, using ';'
+\startpipeline
+SELECT 'val1';
+copy psql_pipeline TO STDOUT;
+\syncpipeline
+\getresults
+\endpipeline
+
--
-- Pipeline errors
--
@@ -217,13 +336,6 @@ SELECT $1 \bind 3 \sendpipeline
-- \endpipeline outside of pipeline should fail
\endpipeline
--- Query using simple protocol should not be sent and should leave the
--- pipeline usable.
-\startpipeline
-SELECT 1;
-SELECT $1 \bind 'val1' \sendpipeline
-\endpipeline
-
-- After an aborted pipeline, commands after a \syncpipeline should be
-- displayed.
\startpipeline
@@ -239,6 +351,13 @@ SELECT \bind 'val1' \sendpipeline
SELECT $1 \bind 'val1' \sendpipeline
\endpipeline
+-- Using ';' with a parameter will trigger an incorrect parameter error and
+-- abort the pipeline
+\startpipeline
+SELECT $1;
+SELECT 1;
+\endpipeline
+
-- An explicit transaction with an error needs to be rollbacked after
-- the pipeline.
\startpipeline
@@ -248,7 +367,7 @@ ROLLBACK \bind \sendpipeline
\endpipeline
ROLLBACK;
--- \watch sends a simple query, something not allowed within a pipeline.
+-- \watch is not allowed within a pipeline.
\startpipeline
SELECT \bind \sendpipeline
\watch 1
@@ -372,8 +491,8 @@ select 1;
-- Error messages accumulate and are repeated.
\startpipeline
SELECT 1 \bind \sendpipeline
-SELECT 1;
-SELECT 1;
+\gdesc
+\gdesc
\endpipeline
--
--
2.39.5 (Apple Git-154)
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-18 09:27 Jelte Fennema-Nio <[email protected]>
parent: Anthonin Bonnefoy <[email protected]>
1 sibling, 1 reply; 41+ messages in thread
From: Jelte Fennema-Nio @ 2025-03-18 09:27 UTC (permalink / raw)
To: Anthonin Bonnefoy <[email protected]>; +Cc: Michael Paquier <[email protected]>; Daniel Verite <[email protected]>; pgsql-hackers
On Tue, 18 Mar 2025 at 09:55, Anthonin Bonnefoy
<[email protected]> wrote:
> I've added additional tests when piping queries with ';':
> - I've reused the same scenario with \sendpipeline: single query,
> multiple queries, flushes, syncs, using COPY...
> - Using ';' will replace the unnamed prepared statement. It's a bit
> different from expected as a simple query will delete the unnamed
> prepared statement.
> - Sending an extended query prepared with \bind using a ';' on a
> newline, though this is not specific to pipelining. The scanned
> semicolon triggers the call to SendQuery, processing the buffered
> extended query. It's a bit unusual but that's the current behaviour.
One thing that comes to mind that I think would be quite useful and
pretty easy to implement if we have this functionality within a
pipeline: An \extended command. That puts psql in "extended protocol
mode" (without enabling pipelining). In "extended protocol mode" all
queries would automatically be sent using PQsendQueryParams. That
would remove the need to use \bind anymore outside of a pipeline
either.
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-18 09:36 Daniel Verite <[email protected]>
parent: Anthonin Bonnefoy <[email protected]>
2 siblings, 1 reply; 41+ messages in thread
From: Daniel Verite @ 2025-03-18 09:36 UTC (permalink / raw)
To: Anthonin Bonnefoy <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; Michael Paquier <[email protected]>; pgsql-hackers
Anthonin Bonnefoy wrote:
> 0002: Allows ';' to send a query using extended protocol when within a
> pipeline by using PQsendQueryParams
It's a nice improvement!
> with 0 parameters. It is not
> possible to send parameters with extended protocol this way and
> everything will be propagated through the query string, similar to a
> simple query.
It's actually possible to use parameters
\startpipeline
\bind 'foo'
select $1;
\endpipeline
?column?
----------
foo
(1 row)
I suspect there's a misunderstanding that \bind can only be placed
after the query, because it's always written like that in the regression
tests, and in the documentation.
But that's just a notational preference.
Best regards,
--
Daniel Vérité
https://postgresql.verite.pro/
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-18 23:56 Michael Paquier <[email protected]>
parent: Daniel Verite <[email protected]>
0 siblings, 0 replies; 41+ messages in thread
From: Michael Paquier @ 2025-03-18 23:56 UTC (permalink / raw)
To: Daniel Verite <[email protected]>; +Cc: Anthonin Bonnefoy <[email protected]>; Jelte Fennema-Nio <[email protected]>; pgsql-hackers
On Tue, Mar 18, 2025 at 10:36:28AM +0100, Daniel Verite wrote:
> It's actually possible to use parameters
>
> \startpipeline
> \bind 'foo'
> select $1;
> \endpipeline
>
> ?column?
> ----------
> foo
> (1 row)
>
> I suspect there's a misunderstanding that \bind can only be placed
> after the query, because it's always written like that in the regression
> tests, and in the documentation.
> But that's just a notational preference.
Nice trick, unrelated to pipelines. I don't think that we have
anything testing this specific pattern for \bind. At quick glance all
our test use \bind at the end of a query string. Perhaps we should?
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-19 02:28 Michael Paquier <[email protected]>
parent: Jelte Fennema-Nio <[email protected]>
0 siblings, 1 reply; 41+ messages in thread
From: Michael Paquier @ 2025-03-19 02:28 UTC (permalink / raw)
To: Jelte Fennema-Nio <[email protected]>; +Cc: Anthonin Bonnefoy <[email protected]>; Daniel Verite <[email protected]>; pgsql-hackers
On Tue, Mar 18, 2025 at 10:27:38AM +0100, Jelte Fennema-Nio wrote:
> One thing that comes to mind that I think would be quite useful and
> pretty easy to implement if we have this functionality within a
> pipeline: An \extended command. That puts psql in "extended protocol
> mode" (without enabling pipelining). In "extended protocol mode" all
> queries would automatically be sent using PQsendQueryParams. That
> would remove the need to use \bind anymore outside of a pipeline
> either.
How does that help when passing parameter values? \bind is here to be
able to pass down parameter values to queries that are prepared, so we
cannot bypass it as the parameter values need to be passed to the
\bind meta-command itself.
Perhaps an \extended command that behaves outside a pipeline makes
sense to force the use of queries without parameters to use the
extended mode, but I cannot get much excited about the concept knowing
all the meta-commands we have now (not talking about the pipeline
part, which is different, as we can treat queries in batches).
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-19 04:49 Michael Paquier <[email protected]>
parent: Anthonin Bonnefoy <[email protected]>
1 sibling, 0 replies; 41+ messages in thread
From: Michael Paquier @ 2025-03-19 04:49 UTC (permalink / raw)
To: Anthonin Bonnefoy <[email protected]>; +Cc: Daniel Verite <[email protected]>; Jelte Fennema-Nio <[email protected]>; pgsql-hackers
On Tue, Mar 18, 2025 at 09:55:21AM +0100, Anthonin Bonnefoy wrote:
> I've added additional tests when piping queries with ';':
> - I've reused the same scenario with \sendpipeline: single query,
> multiple queries, flushes, syncs, using COPY...
> - Using ';' will replace the unnamed prepared statement. It's a bit
> different from expected as a simple query will delete the unnamed
> prepared statement.
> - Sending an extended query prepared with \bind using a ';' on a
> newline, though this is not specific to pipelining. The scanned
> semicolon triggers the call to SendQuery, processing the buffered
> extended query. It's a bit unusual but that's the current behaviour.
The tests could be much more organized, particularly for the "sinple"
and "multiple" and COPY cases, rather than being treated as two
different groups at different locations of psql_pipeline.sql. I've
spent some time reorganizing all that.
A second thing that was a bit itchy is the use of ";" for what's a
semicolon, and we use this term in the psql docs to refer to queries
terminated by that. The whole paragraph could be simplified a bit
more, mentioning that everything in a pipeline uses the extended
protocol, while \bind & co are more like options. The description of
PIPELINE_COMMAND_COUNT could be simpler, and the part about the
pending results can be more general now so I've removed it.
With all that set, I've applied the patch. If you have more
suggestions, please feel free to mention them.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-03-19 13:05 Daniel Verite <[email protected]>
parent: Michael Paquier <[email protected]>
0 siblings, 0 replies; 41+ messages in thread
From: Daniel Verite @ 2025-03-19 13:05 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; Anthonin Bonnefoy <[email protected]>; pgsql-hackers
Michael Paquier wrote:
> Perhaps an \extended command that behaves outside a pipeline makes
> sense to force the use of queries without parameters to use the
> extended mode, but I cannot get much excited about the concept knowing
> all the meta-commands we have now (not talking about the pipeline
> part, which is different, as we can treat queries in batches).
When psql started supporting the extended query protocol, the question
of enabling it more globally rather than query-by-query was discussed
a bit [1]. The idea was to switch to it with a setting or a variable
rather than a metacommand.
Some pros and cons were mentioned in the thread, but on the whole
it was not convincing enough to get implemented.
[1]
https://www.postgresql.org/message-id/e8dd1cd5-0e04-3598-0518-a605159fe314%40enterprisedb.com
Best regards,
--
Daniel Vérité
https://postgresql.verite.pro/
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-04-16 14:31 a.kozhemyakin <[email protected]>
0 siblings, 1 reply; 41+ messages in thread
From: a.kozhemyakin @ 2025-04-16 14:31 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; Daniel Verite <[email protected]>; +Cc: Anthonin Bonnefoy <[email protected]>; Jelte Fennema-Nio <[email protected]>; pgsql-hackers
Hello,
After commit 2cce0fe on master
When executing query:
psql postgres <<EOF
CREATE TABLE psql_pipeline();
\startpipeline
COPY psql_pipeline FROM STDIN;
SELECT 'val1';
\syncpipeline
\getresults
EOF
ERROR:Â unexpected message type 0x50 during COPY from stdin
CONTEXT:Â COPY psql_pipeline, line 1
Pipeline aborted, command did not run
psql: common.c:1510: discardAbortedPipelineResults: Assertion `res ==
((void *)0) || result_status == PGRES_PIPELINE_ABORTED' failed.
Aborted (core dumped)
The psql crashes with the stack trace:
(gdb) bt
#0Â __pthread_kill_implementation (no_tid=0, signo=6,
threadid=<optimized out>) at ./nptl/pthread_kill.c:44
#1Â __pthread_kill_internal (signo=6, threadid=<optimized out>) at
./nptl/pthread_kill.c:78
#2Â __GI___pthread_kill (threadid=<optimized out>, signo=signo@entry=6)
at ./nptl/pthread_kill.c:89
#3Â 0x0000760edd24527e in __GI_raise (sig=sig@entry=6) at
../sysdeps/posix/raise.c:26
#4Â 0x0000760edd2288ff in __GI_abort () at ./stdlib/abort.c:79
#5Â 0x0000760edd22881b in __assert_fail_base (fmt=0x760edd3d01e8
"%s%s%s:%u: %s%sAssertion `%s' failed.\n%n",
   assertion=assertion@entry=0x5ba46ab79850 "res == ((void *)0) ||
result_status == PGRES_PIPELINE_ABORTED", file=file@entry=0x5ba46ab6fcad
"common.c",
   line=line@entry=1510, function=function@entry=0x5ba46ab9c780
<__PRETTY_FUNCTION__.3> "discardAbortedPipelineResults") at
./assert/assert.c:96
#6Â 0x0000760edd23b517 in __assert_fail
(assertion=assertion@entry=0x5ba46ab79850 "res == ((void *)0) ||
result_status == PGRES_PIPELINE_ABORTED",
   file=file@entry=0x5ba46ab6fcad "common.c", line=line@entry=1510,
   function=function@entry=0x5ba46ab9c780 <__PRETTY_FUNCTION__.3>
"discardAbortedPipelineResults") at ./assert/assert.c:105
#7Â 0x00005ba46ab2bd40 in discardAbortedPipelineResults () at common.c:1510
#8Â ExecQueryAndProcessResults (query=query@entry=0x5ba4a2ec1e10 "SELECT
'val1';", elapsed_msec=elapsed_msec@entry=0x7ffeb07262a8,
   svpt_gone_p=svpt_gone_p@entry=0x7ffeb07262a7,
is_watch=is_watch@entry=false, min_rows=min_rows@entry=0,
opt=opt@entry=0x0, printQueryFout=0x0)
   at common.c:1811
#9Â 0x00005ba46ab2983f in SendQuery (query=0x5ba4a2ec1e10 "SELECT
'val1';") at common.c:1212
#10 0x00005ba46ab3f66a in MainLoop (source=source@entry=0x760edd4038e0
<_IO_2_1_stdin_>) at mainloop.c:515
#11 0x00005ba46ab23f2a in process_file (filename=0x0,
use_relative_path=use_relative_path@entry=false) at command.c:4870
#12 0x00005ba46ab1e9d9 in main (argc=<optimized out>,
argv=0x7ffeb07269d8) at startup.c:420
06.03.2025 11:20, Michael Paquier пишет:
> On Wed, Mar 05, 2025 at 03:25:12PM +0100, Daniel Verite wrote:
>> Anthonin Bonnefoy wrote:
>>> I do see the idea to make it easier to convert existing scripts into
>>> using pipelining. The main focus of the initial implementation was
>>> more on protocol regression tests with psql, so that's not necessarily
>>> something I had in mind.
>> Understood. Yet pipelining can accelerate considerably certain scripts
>> when client-server latency is an issue. We should expect end users to
>> benefit from it too.
> That was not a test case we had in mind originally here, but if it is
> possible to keep the implementation simple while supporting your
> demand, well, let's do it. If it's not that straight-forward, let's
> use the new meta-command, forbidding \g and \gx based on your
> arguments from upthread.
> --
> Michael
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-04-16 16:18 Michael Paquier <[email protected]>
parent: a.kozhemyakin <[email protected]>
0 siblings, 1 reply; 41+ messages in thread
From: Michael Paquier @ 2025-04-16 16:18 UTC (permalink / raw)
To: a.kozhemyakin <[email protected]>; +Cc: Daniel Verite <[email protected]>; Anthonin Bonnefoy <[email protected]>; Jelte Fennema-Nio <[email protected]>; pgsql-hackers
On Wed, Apr 16, 2025 at 09:31:59PM +0700, a.kozhemyakin wrote:
> After commit 2cce0fe on master
>
> ERROR:Â unexpected message type 0x50 during COPY from stdin
> CONTEXT:Â COPY psql_pipeline, line 1
> Pipeline aborted, command did not run
> psql: common.c:1510: discardAbortedPipelineResults: Assertion `res == ((void
> *)0) || result_status == PGRES_PIPELINE_ABORTED' failed.
> Aborted (core dumped)
Reproduced here, thanks for the report. I'll look at that at the
beginning of next week, adding an open item for now.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 41+ messages in thread
* Re: Add Pipelining support in psql
@ 2025-04-22 12:37 Anthonin Bonnefoy <[email protected]>
parent: Michael Paquier <[email protected]>
0 siblings, 0 replies; 41+ messages in thread
From: Anthonin Bonnefoy @ 2025-04-22 12:37 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: a.kozhemyakin <[email protected]>; Daniel Verite <[email protected]>; Jelte Fennema-Nio <[email protected]>; pgsql-hackers
On Tue, Apr 22, 2025 at 2:06 AM Michael Paquier <[email protected]> wrote:
> I am wondering if we could not be smarter with the handling of
> the counters, but I really doubt that there is much more we can do
> under a PGRES_FATAL_ERROR.
One thing that bothers me is that the reported error is silently
discarded within discardAbortedPipelineResults.
psql -f bug_assertion.sql
psql:bug_assertion.sql:7: ERROR: unexpected message type 0x50 during
COPY from stdin
CONTEXT: COPY psql_pipeline, line 1
psql:bug_assertion.sql:7: Pipeline aborted, command did not run
This should ideally report the "FATAL: terminating connection because
protocol synchronization was lost" sent by the backend process.
Also, we still have a triggered assertion failure with the following:
CREATE TABLE psql_pipeline(a text);
\startpipeline
COPY psql_pipeline FROM STDIN;
SELECT 'val1';
\syncpipeline
\endpipeline
...
Assertion failed: (pset.piped_syncs == 0), function
ExecQueryAndProcessResults, file common.c, line 2153.
A possible alternative could be to abort discardAbortedPipelineResults
when we encounter a res != NULL + FATAL error and let the outer loop
handle it. As you said, the pipeline flow is borked so there's not
much to salvage. The outer loop would read and print all error
messages until the closed connection is detected. Then,
CheckConnection will reset the connection which will reset the
pipeline state.
While testing this change, I was initially looking for the "FATAL:
terminating connection because protocol synchronization was lost"
message in the tests. However, this was failing on Windows[1] as the
FATAL message wasn't reported on stderr. I'm not sure why yet.
[1]: https://cirrus-ci.com/task/5051031505076224?logs=check_world#L240-L246
Attachments:
[application/octet-stream] v02-0001-PATCH-psql-Fix-assertion-failure-with-pipeline-m.patch (3.7K, ../../CAO6_XqoQaZVHEw1dFgvOxwectqsNo4QE1tQ6rLD-YUzFpT_+3g@mail.gmail.com/2-v02-0001-PATCH-psql-Fix-assertion-failure-with-pipeline-m.patch)
download | inline diff:
From 5d37f2616c82c6525d656149d383ef01a6d7518c Mon Sep 17 00:00:00 2001
From: Michael Paquier <[email protected]>
Date: Mon, 21 Apr 2025 15:17:43 +0900
Subject: [PATCH] psql: Fix assertion failure with pipeline mode
---
src/bin/psql/common.c | 17 ++++++++++++
src/bin/psql/t/001_basic.pl | 55 ++++++++++++++++++++++++++++++++++---
2 files changed, 68 insertions(+), 4 deletions(-)
diff --git a/src/bin/psql/common.c b/src/bin/psql/common.c
index 21d660a8961..0aab02ee32e 100644
--- a/src/bin/psql/common.c
+++ b/src/bin/psql/common.c
@@ -1478,6 +1478,23 @@ discardAbortedPipelineResults(void)
*/
return res;
}
+ else if (res != NULL && result_status == PGRES_FATAL_ERROR)
+ {
+ /*
+ * We have a fatal error sent by the backend and we can't recover
+ * from this state. Instead, return the last fatal error and let
+ * the outer loop handle it.
+ */
+ PGresult *fatal_res PG_USED_FOR_ASSERTS_ONLY;
+
+ /*
+ * Fetch result to consume the end of the current query being
+ * processed.
+ */
+ fatal_res = PQgetResult(pset.db);
+ Assert(fatal_res == NULL);
+ return res;
+ }
else if (res == NULL)
{
/* A query was processed, decrement the counters */
diff --git a/src/bin/psql/t/001_basic.pl b/src/bin/psql/t/001_basic.pl
index 739cb439708..8d258c00c5e 100644
--- a/src/bin/psql/t/001_basic.pl
+++ b/src/bin/psql/t/001_basic.pl
@@ -34,11 +34,13 @@ sub psql_fails_like
{
local $Test::Builder::Level = $Test::Builder::Level + 1;
- my ($node, $sql, $expected_stderr, $test_name) = @_;
+ my ($node, $sql, $expected_stderr, $test_name, $replication) = @_;
+
+ # Use the context of a WAL sender, if requested by the caller.
+ $replication = '' unless defined($replication);
- # Use the context of a WAL sender, some of the tests rely on that.
my ($ret, $stdout, $stderr) =
- $node->psql('postgres', $sql, replication => 'database');
+ $node->psql('postgres', $sql, replication => $replication);
isnt($ret, 0, "$test_name: exit code not 0");
like($stderr, $expected_stderr, "$test_name: matches");
@@ -79,7 +81,8 @@ psql_fails_like(
$node,
'START_REPLICATION 0/0',
qr/unexpected PQresultStatus: 8$/,
- 'handling of unexpected PQresultStatus');
+ 'handling of unexpected PQresultStatus',
+ 'database');
# test \timing
psql_like(
@@ -481,4 +484,48 @@ psql_like($node, "copy (values ('foo'),('bar')) to stdout \\g | $pipe_cmd",
my $c4 = slurp_file($g_file);
like($c4, qr/foo.*bar/s);
+# Tests with pipelines. These trigger FATAL failures in the backend,
+# so they cannot be tested through the SQL regression tests.
+$node->safe_psql('postgres', 'CREATE TABLE psql_pipeline()');
+psql_fails_like(
+ $node,
+ qq{\\startpipeline
+COPY psql_pipeline FROM STDIN;
+SELECT 'val1';
+\\syncpipeline
+\\getresults
+\\endpipeline},
+ qr/server closed the connection unexpectedly/,
+ 'handling of protocol synchronization loss with pipelines');
+psql_fails_like(
+ $node,
+ qq{\\startpipeline
+COPY psql_pipeline FROM STDIN \\bind \\sendpipeline
+SELECT 'val1' \\bind \\sendpipeline
+\\syncpipeline
+\\getresults
+\\endpipeline},
+ qr/server closed the connection unexpectedly/,
+ 'handling of protocol synchronization loss with pipelines');
+
+# This time, test without the \getresults
+psql_fails_like(
+ $node,
+ qq{\\startpipeline
+COPY psql_pipeline FROM STDIN;
+SELECT 'val1';
+\\syncpipeline
+\\endpipeline},
+ qr/server closed the connection unexpectedly/,
+ 'handling of protocol synchronization loss with pipelines');
+psql_fails_like(
+ $node,
+ qq{\\startpipeline
+COPY psql_pipeline FROM STDIN \\bind \\sendpipeline
+SELECT 'val1' \\bind \\sendpipeline
+\\syncpipeline
+\\endpipeline},
+ qr/server closed the connection unexpectedly/,
+ 'handling of protocol synchronization loss with pipelines');
+
done_testing();
--
2.39.5 (Apple Git-154)
^ permalink raw reply [nested|flat] 41+ messages in thread
* [PATCH v53 5/7] Check for transaction block early in ExecRepack
@ 2026-04-03 15:23 Srinath Reddy Sadipiralla <[email protected]>
0 siblings, 0 replies; 41+ messages in thread
From: Srinath Reddy Sadipiralla @ 2026-04-03 15:23 UTC (permalink / raw)
Currently, executing REPACK (CONCURRENTLY) without a table name inside a
transaction block throws the error "REPACK CONCURRENTLY requires explicit
table name" instead of the expected transaction block error. This occurs
because ExecRepack() validates the parsed options and missing relation
before verifying the transaction state.
This behavior is inconsistent with other utility commands like VACUUM
,REINDEX, etc; which invoke PreventInTransactionBlock() at the very start
of their execution to properly reject execution inside user transactions
before validating targets.
Add PreventInTransactionBlock to the top of ExecRepack() to enforce the
transaction block restriction early. This prevents the user from fixing
a missing table error only to immediately hit a transaction block error,
and also ensures consistency with rest of the commands.
Author: Srinath Reddy Sadipiralla <[email protected]>
---
src/backend/commands/repack.c | 27 +++++++++++++++------------
1 file changed, 15 insertions(+), 12 deletions(-)
diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c
index f2759cdbef1..8d9b2b2e370 100644
--- a/src/backend/commands/repack.c
+++ b/src/backend/commands/repack.c
@@ -279,6 +279,21 @@ ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel)
/* Determine the lock mode to use. */
lockmode = RepackLockLevel((params.options & CLUOPT_CONCURRENT) != 0);
+ if ((params.options & CLUOPT_CONCURRENT) != 0)
+ {
+ /*
+ * Make sure we're not in a transaction block.
+ *
+ * The reason is that repack_setup_logical_decoding() could wait
+ * indefinitely for our XID to complete. (The deadlock detector would
+ * not recognize it because we'd be waiting for ourselves, i.e. no
+ * real lock conflict.) It would be possible to run in a transaction
+ * block if we had no XID, but this restriction is simpler for users
+ * to understand and we don't lose any functionality.
+ */
+ PreventInTransactionBlock(isTopLevel, "REPACK (CONCURRENTLY)");
+ }
+
/*
* If a single relation is specified, process it and we're done ... unless
* the relation is a partitioned table, in which case we fall through.
@@ -511,19 +526,7 @@ cluster_rel(RepackCommand cmd, Relation OldHeap, Oid indexOid,
* replica index OID.
*/
if (concurrent)
- {
- /*
- * Make sure we're not in a transaction block.
- *
- * The reason is that repack_setup_logical_decoding() could deadlock
- * if there's an XID already assigned. It would be possible to run in
- * a transaction block if we had no XID, but this restriction is
- * simpler for users to understand and we don't lose anything.
- */
- PreventInTransactionBlock(isTopLevel, "REPACK (CONCURRENTLY)");
-
check_repack_concurrently_requirements(OldHeap, &ident_idx);
- }
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
--
2.47.3
--qr3jlalmmcpkiodg
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
filename="v53-0006-Error-out-any-process-that-would-block-at-REPACK.patch"
^ permalink raw reply [nested|flat] 41+ messages in thread
* [PATCH v51 07/10] Check for transaction block early in ExecRepack
@ 2026-04-03 15:23 Srinath Reddy Sadipiralla <[email protected]>
0 siblings, 0 replies; 41+ messages in thread
From: Srinath Reddy Sadipiralla @ 2026-04-03 15:23 UTC (permalink / raw)
Currently, executing REPACK (CONCURRENTLY) without a table name inside a
transaction block throws the error "REPACK CONCURRENTLY requires explicit
table name" instead of the expected transaction block error. This occurs
because ExecRepack() validates the parsed options and missing relation
before verifying the transaction state.
This behavior is inconsistent with other utility commands like VACUUM
,REINDEX, etc; which invoke PreventInTransactionBlock() at the very start
of their execution to properly reject execution inside user transactions
before validating targets.
Add PreventInTransactionBlock to the top of ExecRepack() to enforce the
transaction block restriction early. This prevents the user from fixing
a missing table error only to immediately hit a transaction block error,
and also ensures consistency with rest of the commands.
Author: Srinath Reddy Sadipiralla <[email protected]>
---
src/backend/commands/repack.c | 26 ++++++++++++--------------
1 file changed, 12 insertions(+), 14 deletions(-)
diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c
index d6e446d582d..d4c1f0e7652 100644
--- a/src/backend/commands/repack.c
+++ b/src/backend/commands/repack.c
@@ -292,6 +292,18 @@ ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel)
MyProc->statusFlags |= PROC_IN_CONCURRENT_REPACK;
ProcGlobal->statusFlags[MyProc->pgxactoff] = MyProc->statusFlags;
LWLockRelease(ProcArrayLock);
+
+ /*
+ * Make sure we're not in a transaction block.
+ *
+ * The reason is that repack_setup_logical_decoding() could wait
+ * indefinitely for our XID to complete. (The deadlock detector would
+ * not recognize it because we'd be waiting for ourselves, i.e. no
+ * real lock conflict.) It would be possible to run in a transaction
+ * block if we had no XID, but this restriction is simpler for users
+ * to understand and we don't lose anything.
+ */
+ PreventInTransactionBlock(isTopLevel, "REPACK (CONCURRENTLY)");
}
/*
@@ -523,21 +535,7 @@ cluster_rel(RepackCommand cmd, Relation OldHeap, Oid indexOid,
* replica index OID.
*/
if (concurrent)
- {
- /*
- * Make sure we're not in a transaction block.
- *
- * The reason is that repack_setup_logical_decoding() could wait
- * indefinitely for our XID to complete. (The deadlock detector would
- * not recognize it because we'd be waiting for ourselves, i.e. no
- * real lock conflict.) It would be possible to run in a transaction
- * block if we had no XID, but this restriction is simpler for users
- * to understand and we don't lose anything.
- */
- PreventInTransactionBlock(isTopLevel, "REPACK (CONCURRENTLY)");
-
check_repack_concurrently_requirements(OldHeap, &ident_idx);
- }
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
--
2.47.3
--r2slln3zpmilwu22
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
filename="v51-0008-Introduce-an-option-to-make-logical-replication-.patch"
^ permalink raw reply [nested|flat] 41+ messages in thread
* [PATCH v51 07/10] Check for transaction block early in ExecRepack
@ 2026-04-03 15:23 Srinath Reddy Sadipiralla <[email protected]>
0 siblings, 0 replies; 41+ messages in thread
From: Srinath Reddy Sadipiralla @ 2026-04-03 15:23 UTC (permalink / raw)
Currently, executing REPACK (CONCURRENTLY) without a table name inside a
transaction block throws the error "REPACK CONCURRENTLY requires explicit
table name" instead of the expected transaction block error. This occurs
because ExecRepack() validates the parsed options and missing relation
before verifying the transaction state.
This behavior is inconsistent with other utility commands like VACUUM
,REINDEX, etc; which invoke PreventInTransactionBlock() at the very start
of their execution to properly reject execution inside user transactions
before validating targets.
Add PreventInTransactionBlock to the top of ExecRepack() to enforce the
transaction block restriction early. This prevents the user from fixing
a missing table error only to immediately hit a transaction block error,
and also ensures consistency with rest of the commands.
Author: Srinath Reddy Sadipiralla <[email protected]>
---
src/backend/commands/repack.c | 26 ++++++++++++--------------
1 file changed, 12 insertions(+), 14 deletions(-)
diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c
index d6e446d582d..d4c1f0e7652 100644
--- a/src/backend/commands/repack.c
+++ b/src/backend/commands/repack.c
@@ -292,6 +292,18 @@ ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel)
MyProc->statusFlags |= PROC_IN_CONCURRENT_REPACK;
ProcGlobal->statusFlags[MyProc->pgxactoff] = MyProc->statusFlags;
LWLockRelease(ProcArrayLock);
+
+ /*
+ * Make sure we're not in a transaction block.
+ *
+ * The reason is that repack_setup_logical_decoding() could wait
+ * indefinitely for our XID to complete. (The deadlock detector would
+ * not recognize it because we'd be waiting for ourselves, i.e. no
+ * real lock conflict.) It would be possible to run in a transaction
+ * block if we had no XID, but this restriction is simpler for users
+ * to understand and we don't lose anything.
+ */
+ PreventInTransactionBlock(isTopLevel, "REPACK (CONCURRENTLY)");
}
/*
@@ -523,21 +535,7 @@ cluster_rel(RepackCommand cmd, Relation OldHeap, Oid indexOid,
* replica index OID.
*/
if (concurrent)
- {
- /*
- * Make sure we're not in a transaction block.
- *
- * The reason is that repack_setup_logical_decoding() could wait
- * indefinitely for our XID to complete. (The deadlock detector would
- * not recognize it because we'd be waiting for ourselves, i.e. no
- * real lock conflict.) It would be possible to run in a transaction
- * block if we had no XID, but this restriction is simpler for users
- * to understand and we don't lose anything.
- */
- PreventInTransactionBlock(isTopLevel, "REPACK (CONCURRENTLY)");
-
check_repack_concurrently_requirements(OldHeap, &ident_idx);
- }
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
--
2.47.3
--r2slln3zpmilwu22
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
filename="v51-0008-Introduce-an-option-to-make-logical-replication-.patch"
^ permalink raw reply [nested|flat] 41+ messages in thread
* [PATCH v53 5/7] Check for transaction block early in ExecRepack
@ 2026-04-03 15:23 Srinath Reddy Sadipiralla <[email protected]>
0 siblings, 0 replies; 41+ messages in thread
From: Srinath Reddy Sadipiralla @ 2026-04-03 15:23 UTC (permalink / raw)
Currently, executing REPACK (CONCURRENTLY) without a table name inside a
transaction block throws the error "REPACK CONCURRENTLY requires explicit
table name" instead of the expected transaction block error. This occurs
because ExecRepack() validates the parsed options and missing relation
before verifying the transaction state.
This behavior is inconsistent with other utility commands like VACUUM
,REINDEX, etc; which invoke PreventInTransactionBlock() at the very start
of their execution to properly reject execution inside user transactions
before validating targets.
Add PreventInTransactionBlock to the top of ExecRepack() to enforce the
transaction block restriction early. This prevents the user from fixing
a missing table error only to immediately hit a transaction block error,
and also ensures consistency with rest of the commands.
Author: Srinath Reddy Sadipiralla <[email protected]>
---
src/backend/commands/repack.c | 27 +++++++++++++++------------
1 file changed, 15 insertions(+), 12 deletions(-)
diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c
index f2759cdbef1..8d9b2b2e370 100644
--- a/src/backend/commands/repack.c
+++ b/src/backend/commands/repack.c
@@ -279,6 +279,21 @@ ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel)
/* Determine the lock mode to use. */
lockmode = RepackLockLevel((params.options & CLUOPT_CONCURRENT) != 0);
+ if ((params.options & CLUOPT_CONCURRENT) != 0)
+ {
+ /*
+ * Make sure we're not in a transaction block.
+ *
+ * The reason is that repack_setup_logical_decoding() could wait
+ * indefinitely for our XID to complete. (The deadlock detector would
+ * not recognize it because we'd be waiting for ourselves, i.e. no
+ * real lock conflict.) It would be possible to run in a transaction
+ * block if we had no XID, but this restriction is simpler for users
+ * to understand and we don't lose any functionality.
+ */
+ PreventInTransactionBlock(isTopLevel, "REPACK (CONCURRENTLY)");
+ }
+
/*
* If a single relation is specified, process it and we're done ... unless
* the relation is a partitioned table, in which case we fall through.
@@ -511,19 +526,7 @@ cluster_rel(RepackCommand cmd, Relation OldHeap, Oid indexOid,
* replica index OID.
*/
if (concurrent)
- {
- /*
- * Make sure we're not in a transaction block.
- *
- * The reason is that repack_setup_logical_decoding() could deadlock
- * if there's an XID already assigned. It would be possible to run in
- * a transaction block if we had no XID, but this restriction is
- * simpler for users to understand and we don't lose anything.
- */
- PreventInTransactionBlock(isTopLevel, "REPACK (CONCURRENTLY)");
-
check_repack_concurrently_requirements(OldHeap, &ident_idx);
- }
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
--
2.47.3
--qr3jlalmmcpkiodg
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
filename="v53-0006-Error-out-any-process-that-would-block-at-REPACK.patch"
^ permalink raw reply [nested|flat] 41+ messages in thread
* [PATCH v52 07/10] Check for transaction block early in ExecRepack
@ 2026-04-03 15:23 Srinath Reddy Sadipiralla <[email protected]>
0 siblings, 0 replies; 41+ messages in thread
From: Srinath Reddy Sadipiralla @ 2026-04-03 15:23 UTC (permalink / raw)
Currently, executing REPACK (CONCURRENTLY) without a table name inside a
transaction block throws the error "REPACK CONCURRENTLY requires explicit
table name" instead of the expected transaction block error. This occurs
because ExecRepack() validates the parsed options and missing relation
before verifying the transaction state.
This behavior is inconsistent with other utility commands like VACUUM
,REINDEX, etc; which invoke PreventInTransactionBlock() at the very start
of their execution to properly reject execution inside user transactions
before validating targets.
Add PreventInTransactionBlock to the top of ExecRepack() to enforce the
transaction block restriction early. This prevents the user from fixing
a missing table error only to immediately hit a transaction block error,
and also ensures consistency with rest of the commands.
Author: Srinath Reddy Sadipiralla <[email protected]>
---
src/backend/commands/repack.c | 26 ++++++++++++--------------
1 file changed, 12 insertions(+), 14 deletions(-)
diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c
index d6e446d582d..d4c1f0e7652 100644
--- a/src/backend/commands/repack.c
+++ b/src/backend/commands/repack.c
@@ -292,6 +292,18 @@ ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel)
MyProc->statusFlags |= PROC_IN_CONCURRENT_REPACK;
ProcGlobal->statusFlags[MyProc->pgxactoff] = MyProc->statusFlags;
LWLockRelease(ProcArrayLock);
+
+ /*
+ * Make sure we're not in a transaction block.
+ *
+ * The reason is that repack_setup_logical_decoding() could wait
+ * indefinitely for our XID to complete. (The deadlock detector would
+ * not recognize it because we'd be waiting for ourselves, i.e. no
+ * real lock conflict.) It would be possible to run in a transaction
+ * block if we had no XID, but this restriction is simpler for users
+ * to understand and we don't lose anything.
+ */
+ PreventInTransactionBlock(isTopLevel, "REPACK (CONCURRENTLY)");
}
/*
@@ -523,21 +535,7 @@ cluster_rel(RepackCommand cmd, Relation OldHeap, Oid indexOid,
* replica index OID.
*/
if (concurrent)
- {
- /*
- * Make sure we're not in a transaction block.
- *
- * The reason is that repack_setup_logical_decoding() could wait
- * indefinitely for our XID to complete. (The deadlock detector would
- * not recognize it because we'd be waiting for ourselves, i.e. no
- * real lock conflict.) It would be possible to run in a transaction
- * block if we had no XID, but this restriction is simpler for users
- * to understand and we don't lose anything.
- */
- PreventInTransactionBlock(isTopLevel, "REPACK (CONCURRENTLY)");
-
check_repack_concurrently_requirements(OldHeap, &ident_idx);
- }
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
--
2.47.3
--gp2pyozrd5pweboh
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
filename="v52-0008-Introduce-an-option-to-make-logical-replication-.patch"
^ permalink raw reply [nested|flat] 41+ messages in thread
* [PATCH v52 07/10] Check for transaction block early in ExecRepack
@ 2026-04-03 15:23 Srinath Reddy Sadipiralla <[email protected]>
0 siblings, 0 replies; 41+ messages in thread
From: Srinath Reddy Sadipiralla @ 2026-04-03 15:23 UTC (permalink / raw)
Currently, executing REPACK (CONCURRENTLY) without a table name inside a
transaction block throws the error "REPACK CONCURRENTLY requires explicit
table name" instead of the expected transaction block error. This occurs
because ExecRepack() validates the parsed options and missing relation
before verifying the transaction state.
This behavior is inconsistent with other utility commands like VACUUM
,REINDEX, etc; which invoke PreventInTransactionBlock() at the very start
of their execution to properly reject execution inside user transactions
before validating targets.
Add PreventInTransactionBlock to the top of ExecRepack() to enforce the
transaction block restriction early. This prevents the user from fixing
a missing table error only to immediately hit a transaction block error,
and also ensures consistency with rest of the commands.
Author: Srinath Reddy Sadipiralla <[email protected]>
---
src/backend/commands/repack.c | 26 ++++++++++++--------------
1 file changed, 12 insertions(+), 14 deletions(-)
diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c
index d6e446d582d..d4c1f0e7652 100644
--- a/src/backend/commands/repack.c
+++ b/src/backend/commands/repack.c
@@ -292,6 +292,18 @@ ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel)
MyProc->statusFlags |= PROC_IN_CONCURRENT_REPACK;
ProcGlobal->statusFlags[MyProc->pgxactoff] = MyProc->statusFlags;
LWLockRelease(ProcArrayLock);
+
+ /*
+ * Make sure we're not in a transaction block.
+ *
+ * The reason is that repack_setup_logical_decoding() could wait
+ * indefinitely for our XID to complete. (The deadlock detector would
+ * not recognize it because we'd be waiting for ourselves, i.e. no
+ * real lock conflict.) It would be possible to run in a transaction
+ * block if we had no XID, but this restriction is simpler for users
+ * to understand and we don't lose anything.
+ */
+ PreventInTransactionBlock(isTopLevel, "REPACK (CONCURRENTLY)");
}
/*
@@ -523,21 +535,7 @@ cluster_rel(RepackCommand cmd, Relation OldHeap, Oid indexOid,
* replica index OID.
*/
if (concurrent)
- {
- /*
- * Make sure we're not in a transaction block.
- *
- * The reason is that repack_setup_logical_decoding() could wait
- * indefinitely for our XID to complete. (The deadlock detector would
- * not recognize it because we'd be waiting for ourselves, i.e. no
- * real lock conflict.) It would be possible to run in a transaction
- * block if we had no XID, but this restriction is simpler for users
- * to understand and we don't lose anything.
- */
- PreventInTransactionBlock(isTopLevel, "REPACK (CONCURRENTLY)");
-
check_repack_concurrently_requirements(OldHeap, &ident_idx);
- }
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
--
2.47.3
--gp2pyozrd5pweboh
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
filename="v52-0008-Introduce-an-option-to-make-logical-replication-.patch"
^ permalink raw reply [nested|flat] 41+ messages in thread
end of thread, other threads:[~2026-04-03 15:23 UTC | newest]
Thread overview: 41+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2019-09-26 21:05 [PATCH v1 05/12] jit: explain: remove backend lifetime module count from function name. Andres Freund <[email protected]>
2019-09-26 21:05 [PATCH v2 5/8] jit: explain: remove backend lifetime module count from function name. Andres Freund <[email protected]>
2022-11-19 16:31 [PATCH 15/16] Rename header comment and #define Justin Pryzby <[email protected]>
2024-12-10 10:43 Re: Add Pipelining support in psql Anthonin Bonnefoy <[email protected]>
2024-12-11 23:53 ` Re: Add Pipelining support in psql Jelte Fennema-Nio <[email protected]>
2024-12-12 10:38 ` Re: Add Pipelining support in psql Anthonin Bonnefoy <[email protected]>
2025-01-10 10:59 ` Re: Add Pipelining support in psql Anthonin Bonnefoy <[email protected]>
2025-01-14 08:49 ` Re: Add Pipelining support in psql Anthonin Bonnefoy <[email protected]>
2025-02-18 07:22 ` Re: Add Pipelining support in psql Michael Paquier <[email protected]>
2025-02-18 17:34 ` Re: Add Pipelining support in psql Anthonin Bonnefoy <[email protected]>
2025-02-20 08:02 ` Re: Add Pipelining support in psql Michael Paquier <[email protected]>
2025-02-20 09:29 ` Re: Add Pipelining support in psql Anthonin Bonnefoy <[email protected]>
2025-02-21 02:33 ` Re: Add Pipelining support in psql Michael Paquier <[email protected]>
2025-02-25 01:11 ` Re: Add Pipelining support in psql Michael Paquier <[email protected]>
2025-02-25 07:30 ` Re: Add Pipelining support in psql Anthonin Bonnefoy <[email protected]>
2025-03-07 00:05 ` Re: Add Pipelining support in psql Jelte Fennema-Nio <[email protected]>
2025-03-07 08:08 ` Re: Add Pipelining support in psql Anthonin Bonnefoy <[email protected]>
2025-03-07 22:31 ` Re: Add Pipelining support in psql Daniel Verite <[email protected]>
2025-03-17 09:50 ` Re: Add Pipelining support in psql Anthonin Bonnefoy <[email protected]>
2025-03-17 10:19 ` Re: Add Pipelining support in psql Michael Paquier <[email protected]>
2025-03-18 00:50 ` Re: Add Pipelining support in psql Michael Paquier <[email protected]>
2025-03-18 08:55 ` Re: Add Pipelining support in psql Anthonin Bonnefoy <[email protected]>
2025-03-18 09:27 ` Re: Add Pipelining support in psql Jelte Fennema-Nio <[email protected]>
2025-03-19 02:28 ` Re: Add Pipelining support in psql Michael Paquier <[email protected]>
2025-03-19 13:05 ` Re: Add Pipelining support in psql Daniel Verite <[email protected]>
2025-03-19 04:49 ` Re: Add Pipelining support in psql Michael Paquier <[email protected]>
2025-03-18 09:36 ` Re: Add Pipelining support in psql Daniel Verite <[email protected]>
2025-03-18 23:56 ` Re: Add Pipelining support in psql Michael Paquier <[email protected]>
2025-02-28 16:31 ` Re: Add Pipelining support in psql Daniel Verite <[email protected]>
2025-03-04 00:56 ` Re: Add Pipelining support in psql Michael Paquier <[email protected]>
2025-03-04 09:31 ` Re: Add Pipelining support in psql Anthonin Bonnefoy <[email protected]>
2025-03-04 23:11 ` Re: Add Pipelining support in psql Michael Paquier <[email protected]>
2025-04-16 14:31 Re: Add Pipelining support in psql a.kozhemyakin <[email protected]>
2025-04-16 16:18 ` Re: Add Pipelining support in psql Michael Paquier <[email protected]>
2025-04-22 12:37 ` Re: Add Pipelining support in psql Anthonin Bonnefoy <[email protected]>
2026-04-03 15:23 [PATCH v53 5/7] Check for transaction block early in ExecRepack Srinath Reddy Sadipiralla <[email protected]>
2026-04-03 15:23 [PATCH v52 07/10] Check for transaction block early in ExecRepack Srinath Reddy Sadipiralla <[email protected]>
2026-04-03 15:23 [PATCH v52 07/10] Check for transaction block early in ExecRepack Srinath Reddy Sadipiralla <[email protected]>
2026-04-03 15:23 [PATCH v51 07/10] Check for transaction block early in ExecRepack Srinath Reddy Sadipiralla <[email protected]>
2026-04-03 15:23 [PATCH v51 07/10] Check for transaction block early in ExecRepack Srinath Reddy Sadipiralla <[email protected]>
2026-04-03 15:23 [PATCH v53 5/7] Check for transaction block early in ExecRepack Srinath Reddy Sadipiralla <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox