public inbox for [email protected]
help / color / mirror / Atom feed[PATCH v7 1/1] Introduce macros for protocol characters.
46+ messages / 7 participants
[nested] [flat]
* [PATCH v7 1/1] Introduce macros for protocol characters.
@ 2023-08-17 16:00 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 46+ messages in thread
From: Nathan Bossart @ 2023-08-17 16:00 UTC (permalink / raw)
Author: Dave Cramer
Reviewed-by: Alvaro Herrera, Tatsuo Ishii, Peter Smith, Robert Haas, Tom Lane, Peter Eisentraut, Michael Paquier
Discussion: https://postgr.es/m/CADK3HHKbBmK-PKf1bPNFoMC%2BoBt%2BpD9PH8h5nvmBQskEHm-Ehw%40mail.gmail.com
---
src/backend/access/common/printsimple.c | 5 +-
src/backend/access/transam/parallel.c | 14 ++--
src/backend/backup/basebackup_copy.c | 16 ++---
src/backend/commands/async.c | 2 +-
src/backend/commands/copyfromparse.c | 22 +++----
src/backend/commands/copyto.c | 6 +-
src/backend/libpq/auth-sasl.c | 2 +-
src/backend/libpq/auth.c | 8 +--
src/backend/postmaster/postmaster.c | 2 +-
src/backend/replication/walsender.c | 18 +++---
src/backend/tcop/dest.c | 8 +--
src/backend/tcop/fastpath.c | 2 +-
src/backend/tcop/postgres.c | 68 ++++++++++----------
src/backend/utils/error/elog.c | 5 +-
src/backend/utils/misc/guc.c | 2 +-
src/include/Makefile | 3 +-
src/include/libpq/pqcomm.h | 23 ++-----
src/include/libpq/protocol.h | 85 +++++++++++++++++++++++++
src/include/meson.build | 1 +
src/interfaces/libpq/fe-auth.c | 2 +-
src/interfaces/libpq/fe-connect.c | 19 ++++--
src/interfaces/libpq/fe-exec.c | 54 ++++++++--------
src/interfaces/libpq/fe-protocol3.c | 70 ++++++++++----------
src/interfaces/libpq/fe-trace.c | 70 +++++++++++---------
src/tools/msvc/Install.pm | 2 +
25 files changed, 305 insertions(+), 204 deletions(-)
create mode 100644 src/include/libpq/protocol.h
diff --git a/src/backend/access/common/printsimple.c b/src/backend/access/common/printsimple.c
index ef818228ac..675b744db2 100644
--- a/src/backend/access/common/printsimple.c
+++ b/src/backend/access/common/printsimple.c
@@ -20,6 +20,7 @@
#include "access/printsimple.h"
#include "catalog/pg_type.h"
+#include "libpq/protocol.h"
#include "libpq/pqformat.h"
#include "utils/builtins.h"
@@ -32,7 +33,7 @@ printsimple_startup(DestReceiver *self, int operation, TupleDesc tupdesc)
StringInfoData buf;
int i;
- pq_beginmessage(&buf, 'T'); /* RowDescription */
+ pq_beginmessage(&buf, PqMsg_RowDescription);
pq_sendint16(&buf, tupdesc->natts);
for (i = 0; i < tupdesc->natts; ++i)
@@ -65,7 +66,7 @@ printsimple(TupleTableSlot *slot, DestReceiver *self)
slot_getallattrs(slot);
/* Prepare and send message */
- pq_beginmessage(&buf, 'D');
+ pq_beginmessage(&buf, PqMsg_DataRow);
pq_sendint16(&buf, tupdesc->natts);
for (i = 0; i < tupdesc->natts; ++i)
diff --git a/src/backend/access/transam/parallel.c b/src/backend/access/transam/parallel.c
index 1738aecf1f..194a1207be 100644
--- a/src/backend/access/transam/parallel.c
+++ b/src/backend/access/transam/parallel.c
@@ -1127,7 +1127,7 @@ HandleParallelMessage(ParallelContext *pcxt, int i, StringInfo msg)
switch (msgtype)
{
- case 'K': /* BackendKeyData */
+ case PqMsg_BackendKeyData:
{
int32 pid = pq_getmsgint(msg, 4);
@@ -1137,8 +1137,8 @@ HandleParallelMessage(ParallelContext *pcxt, int i, StringInfo msg)
break;
}
- case 'E': /* ErrorResponse */
- case 'N': /* NoticeResponse */
+ case PqMsg_ErrorResponse:
+ case PqMsg_NoticeResponse:
{
ErrorData edata;
ErrorContextCallback *save_error_context_stack;
@@ -1183,7 +1183,7 @@ HandleParallelMessage(ParallelContext *pcxt, int i, StringInfo msg)
break;
}
- case 'A': /* NotifyResponse */
+ case PqMsg_NotificationResponse:
{
/* Propagate NotifyResponse. */
int32 pid;
@@ -1217,7 +1217,7 @@ HandleParallelMessage(ParallelContext *pcxt, int i, StringInfo msg)
break;
}
- case 'X': /* Terminate, indicating clean exit */
+ case PqMsg_Terminate:
{
shm_mq_detach(pcxt->worker[i].error_mqh);
pcxt->worker[i].error_mqh = NULL;
@@ -1372,7 +1372,7 @@ ParallelWorkerMain(Datum main_arg)
* protocol message is defined, but it won't actually be used for anything
* in this case.
*/
- pq_beginmessage(&msgbuf, 'K');
+ pq_beginmessage(&msgbuf, PqMsg_BackendKeyData);
pq_sendint32(&msgbuf, (int32) MyProcPid);
pq_sendint32(&msgbuf, (int32) MyCancelKey);
pq_endmessage(&msgbuf);
@@ -1550,7 +1550,7 @@ ParallelWorkerMain(Datum main_arg)
DetachSession();
/* Report success. */
- pq_putmessage('X', NULL, 0);
+ pq_putmessage(PqMsg_Terminate, NULL, 0);
}
/*
diff --git a/src/backend/backup/basebackup_copy.c b/src/backend/backup/basebackup_copy.c
index 1db80cde1b..fee30c21e1 100644
--- a/src/backend/backup/basebackup_copy.c
+++ b/src/backend/backup/basebackup_copy.c
@@ -152,7 +152,7 @@ bbsink_copystream_begin_backup(bbsink *sink)
SendTablespaceList(state->tablespaces);
/* Send a CommandComplete message */
- pq_puttextmessage('C', "SELECT");
+ pq_puttextmessage(PqMsg_CommandComplete, "SELECT");
/* Begin COPY stream. This will be used for all archives + manifest. */
SendCopyOutResponse();
@@ -169,7 +169,7 @@ bbsink_copystream_begin_archive(bbsink *sink, const char *archive_name)
StringInfoData buf;
ti = list_nth(state->tablespaces, state->tablespace_num);
- pq_beginmessage(&buf, 'd'); /* CopyData */
+ pq_beginmessage(&buf, PqMsg_CopyData);
pq_sendbyte(&buf, 'n'); /* New archive */
pq_sendstring(&buf, archive_name);
pq_sendstring(&buf, ti->path == NULL ? "" : ti->path);
@@ -220,7 +220,7 @@ bbsink_copystream_archive_contents(bbsink *sink, size_t len)
{
mysink->last_progress_report_time = now;
- pq_beginmessage(&buf, 'd'); /* CopyData */
+ pq_beginmessage(&buf, PqMsg_CopyData);
pq_sendbyte(&buf, 'p'); /* Progress report */
pq_sendint64(&buf, state->bytes_done);
pq_endmessage(&buf);
@@ -246,7 +246,7 @@ bbsink_copystream_end_archive(bbsink *sink)
mysink->bytes_done_at_last_time_check = state->bytes_done;
mysink->last_progress_report_time = GetCurrentTimestamp();
- pq_beginmessage(&buf, 'd'); /* CopyData */
+ pq_beginmessage(&buf, PqMsg_CopyData);
pq_sendbyte(&buf, 'p'); /* Progress report */
pq_sendint64(&buf, state->bytes_done);
pq_endmessage(&buf);
@@ -261,7 +261,7 @@ bbsink_copystream_begin_manifest(bbsink *sink)
{
StringInfoData buf;
- pq_beginmessage(&buf, 'd'); /* CopyData */
+ pq_beginmessage(&buf, PqMsg_CopyData);
pq_sendbyte(&buf, 'm'); /* Manifest */
pq_endmessage(&buf);
}
@@ -318,7 +318,7 @@ SendCopyOutResponse(void)
{
StringInfoData buf;
- pq_beginmessage(&buf, 'H');
+ pq_beginmessage(&buf, PqMsg_CopyOutResponse);
pq_sendbyte(&buf, 0); /* overall format */
pq_sendint16(&buf, 0); /* natts */
pq_endmessage(&buf);
@@ -330,7 +330,7 @@ SendCopyOutResponse(void)
static void
SendCopyDone(void)
{
- pq_putemptymessage('c');
+ pq_putemptymessage(PqMsg_CopyDone);
}
/*
@@ -368,7 +368,7 @@ SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli)
end_tup_output(tstate);
/* Send a CommandComplete message */
- pq_puttextmessage('C', "SELECT");
+ pq_puttextmessage(PqMsg_CommandComplete, "SELECT");
}
/*
diff --git a/src/backend/commands/async.c b/src/backend/commands/async.c
index ef909cf4e0..d148d10850 100644
--- a/src/backend/commands/async.c
+++ b/src/backend/commands/async.c
@@ -2281,7 +2281,7 @@ NotifyMyFrontEnd(const char *channel, const char *payload, int32 srcPid)
{
StringInfoData buf;
- pq_beginmessage(&buf, 'A');
+ pq_beginmessage(&buf, PqMsg_NotificationResponse);
pq_sendint32(&buf, srcPid);
pq_sendstring(&buf, channel);
pq_sendstring(&buf, payload);
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 232768a6e1..f553734582 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -174,7 +174,7 @@ ReceiveCopyBegin(CopyFromState cstate)
int16 format = (cstate->opts.binary ? 1 : 0);
int i;
- pq_beginmessage(&buf, 'G');
+ pq_beginmessage(&buf, PqMsg_CopyInResponse);
pq_sendbyte(&buf, format); /* overall format */
pq_sendint16(&buf, natts);
for (i = 0; i < natts; i++)
@@ -279,13 +279,13 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
/* Validate message type and set packet size limit */
switch (mtype)
{
- case 'd': /* CopyData */
+ case PqMsg_CopyData:
maxmsglen = PQ_LARGE_MESSAGE_LIMIT;
break;
- case 'c': /* CopyDone */
- case 'f': /* CopyFail */
- case 'H': /* Flush */
- case 'S': /* Sync */
+ case PqMsg_CopyDone:
+ case PqMsg_CopyFail:
+ case PqMsg_Flush:
+ case PqMsg_Sync:
maxmsglen = PQ_SMALL_MESSAGE_LIMIT;
break;
default:
@@ -305,20 +305,20 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
/* ... and process it */
switch (mtype)
{
- case 'd': /* CopyData */
+ case PqMsg_CopyData:
break;
- case 'c': /* CopyDone */
+ case PqMsg_CopyDone:
/* COPY IN correctly terminated by frontend */
cstate->raw_reached_eof = true;
return bytesread;
- case 'f': /* CopyFail */
+ case PqMsg_CopyFail:
ereport(ERROR,
(errcode(ERRCODE_QUERY_CANCELED),
errmsg("COPY from stdin failed: %s",
pq_getmsgstring(cstate->fe_msgbuf))));
break;
- case 'H': /* Flush */
- case 'S': /* Sync */
+ case PqMsg_Flush:
+ case PqMsg_Sync:
/*
* Ignore Flush/Sync for the convenience of client
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 9e4b2437a5..eaa3172793 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -144,7 +144,7 @@ SendCopyBegin(CopyToState cstate)
int16 format = (cstate->opts.binary ? 1 : 0);
int i;
- pq_beginmessage(&buf, 'H');
+ pq_beginmessage(&buf, PqMsg_CopyOutResponse);
pq_sendbyte(&buf, format); /* overall format */
pq_sendint16(&buf, natts);
for (i = 0; i < natts; i++)
@@ -159,7 +159,7 @@ SendCopyEnd(CopyToState cstate)
/* Shouldn't have any unsent data */
Assert(cstate->fe_msgbuf->len == 0);
/* Send Copy Done message */
- pq_putemptymessage('c');
+ pq_putemptymessage(PqMsg_CopyDone);
}
/*----------
@@ -247,7 +247,7 @@ CopySendEndOfRow(CopyToState cstate)
CopySendChar(cstate, '\n');
/* Dump the accumulated row as one CopyData message */
- (void) pq_putmessage('d', fe_msgbuf->data, fe_msgbuf->len);
+ (void) pq_putmessage(PqMsg_CopyData, fe_msgbuf->data, fe_msgbuf->len);
break;
case COPY_CALLBACK:
cstate->data_dest_cb(fe_msgbuf->data, fe_msgbuf->len);
diff --git a/src/backend/libpq/auth-sasl.c b/src/backend/libpq/auth-sasl.c
index 684680897b..c535bc5383 100644
--- a/src/backend/libpq/auth-sasl.c
+++ b/src/backend/libpq/auth-sasl.c
@@ -87,7 +87,7 @@ CheckSASLAuth(const pg_be_sasl_mech *mech, Port *port, char *shadow_pass,
{
pq_startmsgread();
mtype = pq_getbyte();
- if (mtype != 'p')
+ if (mtype != PqMsg_SASLResponse)
{
/* Only log error if client didn't disconnect. */
if (mtype != EOF)
diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c
index 315a24bb3f..0356fe3e45 100644
--- a/src/backend/libpq/auth.c
+++ b/src/backend/libpq/auth.c
@@ -665,7 +665,7 @@ sendAuthRequest(Port *port, AuthRequest areq, const char *extradata, int extrale
CHECK_FOR_INTERRUPTS();
- pq_beginmessage(&buf, 'R');
+ pq_beginmessage(&buf, PqMsg_AuthenticationRequest);
pq_sendint32(&buf, (int32) areq);
if (extralen > 0)
pq_sendbytes(&buf, extradata, extralen);
@@ -698,7 +698,7 @@ recv_password_packet(Port *port)
/* Expect 'p' message type */
mtype = pq_getbyte();
- if (mtype != 'p')
+ if (mtype != PqMsg_PasswordMessage)
{
/*
* If the client just disconnects without offering a password, don't
@@ -961,7 +961,7 @@ pg_GSS_recvauth(Port *port)
CHECK_FOR_INTERRUPTS();
mtype = pq_getbyte();
- if (mtype != 'p')
+ if (mtype != PqMsg_GSSResponse)
{
/* Only log error if client didn't disconnect. */
if (mtype != EOF)
@@ -1232,7 +1232,7 @@ pg_SSPI_recvauth(Port *port)
{
pq_startmsgread();
mtype = pq_getbyte();
- if (mtype != 'p')
+ if (mtype != PqMsg_GSSResponse)
{
if (sspictx != NULL)
{
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index 9c8ec779f9..07d376d77e 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -2357,7 +2357,7 @@ SendNegotiateProtocolVersion(List *unrecognized_protocol_options)
StringInfoData buf;
ListCell *lc;
- pq_beginmessage(&buf, 'v'); /* NegotiateProtocolVersion */
+ pq_beginmessage(&buf, PqMsg_NegotiateProtocolVersion);
pq_sendint32(&buf, PG_PROTOCOL_LATEST);
pq_sendint32(&buf, list_length(unrecognized_protocol_options));
foreach(lc, unrecognized_protocol_options)
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index d27ef2985d..80374c55be 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -603,7 +603,7 @@ SendTimeLineHistory(TimeLineHistoryCmd *cmd)
dest->rStartup(dest, CMD_SELECT, tupdesc);
/* Send a DataRow message */
- pq_beginmessage(&buf, 'D');
+ pq_beginmessage(&buf, PqMsg_DataRow);
pq_sendint16(&buf, 2); /* # of columns */
len = strlen(histfname);
pq_sendint32(&buf, len); /* col1 len */
@@ -801,7 +801,7 @@ StartReplication(StartReplicationCmd *cmd)
WalSndSetState(WALSNDSTATE_CATCHUP);
/* Send a CopyBothResponse message, and start streaming */
- pq_beginmessage(&buf, 'W');
+ pq_beginmessage(&buf, PqMsg_CopyBothResponse);
pq_sendbyte(&buf, 0);
pq_sendint16(&buf, 0);
pq_endmessage(&buf);
@@ -1294,7 +1294,7 @@ StartLogicalReplication(StartReplicationCmd *cmd)
WalSndSetState(WALSNDSTATE_CATCHUP);
/* Send a CopyBothResponse message, and start streaming */
- pq_beginmessage(&buf, 'W');
+ pq_beginmessage(&buf, PqMsg_CopyBothResponse);
pq_sendbyte(&buf, 0);
pq_sendint16(&buf, 0);
pq_endmessage(&buf);
@@ -1923,11 +1923,11 @@ ProcessRepliesIfAny(void)
/* Validate message type and set packet size limit */
switch (firstchar)
{
- case 'd':
+ case PqMsg_CopyData:
maxmsglen = PQ_LARGE_MESSAGE_LIMIT;
break;
- case 'c':
- case 'X':
+ case PqMsg_CopyDone:
+ case PqMsg_Terminate:
maxmsglen = PQ_SMALL_MESSAGE_LIMIT;
break;
default:
@@ -1955,7 +1955,7 @@ ProcessRepliesIfAny(void)
/*
* 'd' means a standby reply wrapped in a CopyData packet.
*/
- case 'd':
+ case PqMsg_CopyData:
ProcessStandbyMessage();
received = true;
break;
@@ -1964,7 +1964,7 @@ ProcessRepliesIfAny(void)
* CopyDone means the standby requested to finish streaming.
* Reply with CopyDone, if we had not sent that already.
*/
- case 'c':
+ case PqMsg_CopyDone:
if (!streamingDoneSending)
{
pq_putmessage_noblock('c', NULL, 0);
@@ -1978,7 +1978,7 @@ ProcessRepliesIfAny(void)
/*
* 'X' means that the standby is closing down the socket.
*/
- case 'X':
+ case PqMsg_Terminate:
proc_exit(0);
default:
diff --git a/src/backend/tcop/dest.c b/src/backend/tcop/dest.c
index c0406e2ee5..06d1872b9a 100644
--- a/src/backend/tcop/dest.c
+++ b/src/backend/tcop/dest.c
@@ -176,7 +176,7 @@ EndCommand(const QueryCompletion *qc, CommandDest dest, bool force_undecorated_o
len = BuildQueryCompletionString(completionTag, qc,
force_undecorated_output);
- pq_putmessage('C', completionTag, len + 1);
+ pq_putmessage(PqMsg_Close, completionTag, len + 1);
case DestNone:
case DestDebug:
@@ -200,7 +200,7 @@ EndCommand(const QueryCompletion *qc, CommandDest dest, bool force_undecorated_o
void
EndReplicationCommand(const char *commandTag)
{
- pq_putmessage('C', commandTag, strlen(commandTag) + 1);
+ pq_putmessage(PqMsg_Close, commandTag, strlen(commandTag) + 1);
}
/* ----------------
@@ -220,7 +220,7 @@ NullCommand(CommandDest dest)
case DestRemoteSimple:
/* Tell the FE that we saw an empty query string */
- pq_putemptymessage('I');
+ pq_putemptymessage(PqMsg_EmptyQueryResponse);
break;
case DestNone:
@@ -258,7 +258,7 @@ ReadyForQuery(CommandDest dest)
{
StringInfoData buf;
- pq_beginmessage(&buf, 'Z');
+ pq_beginmessage(&buf, PqMsg_ReadyForQuery);
pq_sendbyte(&buf, TransactionBlockStatusCode());
pq_endmessage(&buf);
}
diff --git a/src/backend/tcop/fastpath.c b/src/backend/tcop/fastpath.c
index 2f70ebd5fa..71f161dbe2 100644
--- a/src/backend/tcop/fastpath.c
+++ b/src/backend/tcop/fastpath.c
@@ -69,7 +69,7 @@ SendFunctionResult(Datum retval, bool isnull, Oid rettype, int16 format)
{
StringInfoData buf;
- pq_beginmessage(&buf, 'V');
+ pq_beginmessage(&buf, PqMsg_FunctionCallResponse);
if (isnull)
{
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index 36cc99ec9c..e4756f8be2 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -402,37 +402,37 @@ SocketBackend(StringInfo inBuf)
*/
switch (qtype)
{
- case 'Q': /* simple query */
+ case PqMsg_Query:
maxmsglen = PQ_LARGE_MESSAGE_LIMIT;
doing_extended_query_message = false;
break;
- case 'F': /* fastpath function call */
+ case PqMsg_FunctionCall:
maxmsglen = PQ_LARGE_MESSAGE_LIMIT;
doing_extended_query_message = false;
break;
- case 'X': /* terminate */
+ case PqMsg_Terminate:
maxmsglen = PQ_SMALL_MESSAGE_LIMIT;
doing_extended_query_message = false;
ignore_till_sync = false;
break;
- case 'B': /* bind */
- case 'P': /* parse */
+ case PqMsg_Bind:
+ case PqMsg_Parse:
maxmsglen = PQ_LARGE_MESSAGE_LIMIT;
doing_extended_query_message = true;
break;
- case 'C': /* close */
- case 'D': /* describe */
- case 'E': /* execute */
- case 'H': /* flush */
+ case PqMsg_Close:
+ case PqMsg_Describe:
+ case PqMsg_Execute:
+ case PqMsg_Flush:
maxmsglen = PQ_SMALL_MESSAGE_LIMIT;
doing_extended_query_message = true;
break;
- case 'S': /* sync */
+ case PqMsg_Sync:
maxmsglen = PQ_SMALL_MESSAGE_LIMIT;
/* stop any active skip-till-Sync */
ignore_till_sync = false;
@@ -440,13 +440,13 @@ SocketBackend(StringInfo inBuf)
doing_extended_query_message = false;
break;
- case 'd': /* copy data */
+ case PqMsg_CopyData:
maxmsglen = PQ_LARGE_MESSAGE_LIMIT;
doing_extended_query_message = false;
break;
- case 'c': /* copy done */
- case 'f': /* copy fail */
+ case PqMsg_CopyDone:
+ case PqMsg_CopyFail:
maxmsglen = PQ_SMALL_MESSAGE_LIMIT;
doing_extended_query_message = false;
break;
@@ -1589,7 +1589,7 @@ exec_parse_message(const char *query_string, /* string to execute */
* Send ParseComplete.
*/
if (whereToSendOutput == DestRemote)
- pq_putemptymessage('1');
+ pq_putemptymessage(PqMsg_ParseComplete);
/*
* Emit duration logging if appropriate.
@@ -2047,7 +2047,7 @@ exec_bind_message(StringInfo input_message)
* Send BindComplete.
*/
if (whereToSendOutput == DestRemote)
- pq_putemptymessage('2');
+ pq_putemptymessage(PqMsg_BindComplete);
/*
* Emit duration logging if appropriate.
@@ -2290,7 +2290,7 @@ exec_execute_message(const char *portal_name, long max_rows)
{
/* Portal run not complete, so send PortalSuspended */
if (whereToSendOutput == DestRemote)
- pq_putemptymessage('s');
+ pq_putemptymessage(PqMsg_PortalSuspended);
/*
* Set XACT_FLAGS_PIPELINING whenever we suspend an Execute message,
@@ -2683,7 +2683,7 @@ exec_describe_statement_message(const char *stmt_name)
NULL);
}
else
- pq_putemptymessage('n'); /* NoData */
+ pq_putemptymessage(PqMsg_NoData);
}
/*
@@ -2736,7 +2736,7 @@ exec_describe_portal_message(const char *portal_name)
FetchPortalTargetList(portal),
portal->formats);
else
- pq_putemptymessage('n'); /* NoData */
+ pq_putemptymessage(PqMsg_NoData);
}
@@ -4239,7 +4239,7 @@ PostgresMain(const char *dbname, const char *username)
{
StringInfoData buf;
- pq_beginmessage(&buf, 'K');
+ pq_beginmessage(&buf, PqMsg_BackendKeyData);
pq_sendint32(&buf, (int32) MyProcPid);
pq_sendint32(&buf, (int32) MyCancelKey);
pq_endmessage(&buf);
@@ -4618,7 +4618,7 @@ PostgresMain(const char *dbname, const char *username)
switch (firstchar)
{
- case 'Q': /* simple query */
+ case PqMsg_Query:
{
const char *query_string;
@@ -4642,7 +4642,7 @@ PostgresMain(const char *dbname, const char *username)
}
break;
- case 'P': /* parse */
+ case PqMsg_Parse:
{
const char *stmt_name;
const char *query_string;
@@ -4672,7 +4672,7 @@ PostgresMain(const char *dbname, const char *username)
}
break;
- case 'B': /* bind */
+ case PqMsg_Bind:
forbidden_in_wal_sender(firstchar);
/* Set statement_timestamp() */
@@ -4687,7 +4687,7 @@ PostgresMain(const char *dbname, const char *username)
/* exec_bind_message does valgrind_report_error_query */
break;
- case 'E': /* execute */
+ case PqMsg_Execute:
{
const char *portal_name;
int max_rows;
@@ -4707,7 +4707,7 @@ PostgresMain(const char *dbname, const char *username)
}
break;
- case 'F': /* fastpath function call */
+ case PqMsg_FunctionCall:
forbidden_in_wal_sender(firstchar);
/* Set statement_timestamp() */
@@ -4742,7 +4742,7 @@ PostgresMain(const char *dbname, const char *username)
send_ready_for_query = true;
break;
- case 'C': /* close */
+ case PqMsg_Close:
{
int close_type;
const char *close_target;
@@ -4782,13 +4782,13 @@ PostgresMain(const char *dbname, const char *username)
}
if (whereToSendOutput == DestRemote)
- pq_putemptymessage('3'); /* CloseComplete */
+ pq_putemptymessage(PqMsg_CloseComplete);
valgrind_report_error_query("CLOSE message");
}
break;
- case 'D': /* describe */
+ case PqMsg_Describe:
{
int describe_type;
const char *describe_target;
@@ -4822,13 +4822,13 @@ PostgresMain(const char *dbname, const char *username)
}
break;
- case 'H': /* flush */
+ case PqMsg_Flush:
pq_getmsgend(&input_message);
if (whereToSendOutput == DestRemote)
pq_flush();
break;
- case 'S': /* sync */
+ case PqMsg_Sync:
pq_getmsgend(&input_message);
finish_xact_command();
valgrind_report_error_query("SYNC message");
@@ -4847,7 +4847,7 @@ PostgresMain(const char *dbname, const char *username)
/* FALLTHROUGH */
- case 'X':
+ case PqMsg_Terminate:
/*
* Reset whereToSendOutput to prevent ereport from attempting
@@ -4865,9 +4865,9 @@ PostgresMain(const char *dbname, const char *username)
*/
proc_exit(0);
- case 'd': /* copy data */
- case 'c': /* copy done */
- case 'f': /* copy fail */
+ case PqMsg_CopyData:
+ case PqMsg_CopyDone:
+ case PqMsg_CopyFail:
/*
* Accept but ignore these messages, per protocol spec; we
@@ -4897,7 +4897,7 @@ forbidden_in_wal_sender(char firstchar)
{
if (am_walsender)
{
- if (firstchar == 'F')
+ if (firstchar == PqMsg_FunctionCall)
ereport(ERROR,
(errcode(ERRCODE_PROTOCOL_VIOLATION),
errmsg("fastpath function calls not supported in a replication connection")));
diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c
index 5898100acb..8e1f3e8521 100644
--- a/src/backend/utils/error/elog.c
+++ b/src/backend/utils/error/elog.c
@@ -3465,7 +3465,10 @@ send_message_to_frontend(ErrorData *edata)
char tbuf[12];
/* 'N' (Notice) is for nonfatal conditions, 'E' is for errors */
- pq_beginmessage(&msgbuf, (edata->elevel < ERROR) ? 'N' : 'E');
+ if (edata->elevel < ERROR)
+ pq_beginmessage(&msgbuf, PqMsg_NoticeResponse);
+ else
+ pq_beginmessage(&msgbuf, PqMsg_ErrorResponse);
sev = error_severity(edata->elevel);
pq_sendbyte(&msgbuf, PG_DIAG_SEVERITY);
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 99bb2fdd19..84e7ad4d90 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -2593,7 +2593,7 @@ ReportGUCOption(struct config_generic *record)
{
StringInfoData msgbuf;
- pq_beginmessage(&msgbuf, 'S');
+ pq_beginmessage(&msgbuf, PqMsg_ParameterStatus);
pq_sendstring(&msgbuf, record->name);
pq_sendstring(&msgbuf, val);
pq_endmessage(&msgbuf);
diff --git a/src/include/Makefile b/src/include/Makefile
index 5d213187e2..2d5242561c 100644
--- a/src/include/Makefile
+++ b/src/include/Makefile
@@ -40,6 +40,7 @@ install: all installdirs
$(INSTALL_DATA) $(srcdir)/port.h '$(DESTDIR)$(includedir_internal)'
$(INSTALL_DATA) $(srcdir)/postgres_fe.h '$(DESTDIR)$(includedir_internal)'
$(INSTALL_DATA) $(srcdir)/libpq/pqcomm.h '$(DESTDIR)$(includedir_internal)/libpq'
+ $(INSTALL_DATA) $(srcdir)/libpq/protocol.h '$(DESTDIR)$(includedir_internal)/libpq'
# These headers are needed for server-side development
$(INSTALL_DATA) pg_config.h '$(DESTDIR)$(includedir_server)'
$(INSTALL_DATA) pg_config_ext.h '$(DESTDIR)$(includedir_server)'
@@ -65,7 +66,7 @@ installdirs:
uninstall:
rm -f $(addprefix '$(DESTDIR)$(includedir)'/, pg_config.h pg_config_ext.h pg_config_os.h pg_config_manual.h postgres_ext.h libpq/libpq-fs.h)
- rm -f $(addprefix '$(DESTDIR)$(includedir_internal)'/, c.h port.h postgres_fe.h libpq/pqcomm.h)
+ rm -f $(addprefix '$(DESTDIR)$(includedir_internal)'/, c.h port.h postgres_fe.h libpq/pqcomm.h libpq/protocol.h)
# heuristic...
rm -rf $(addprefix '$(DESTDIR)$(includedir_server)'/, $(SUBDIRS) *.h)
diff --git a/src/include/libpq/pqcomm.h b/src/include/libpq/pqcomm.h
index 3da00f7983..46a0946b8b 100644
--- a/src/include/libpq/pqcomm.h
+++ b/src/include/libpq/pqcomm.h
@@ -21,6 +21,12 @@
#include <netdb.h>
#include <netinet/in.h>
+/*
+ * The definitions for the request/response codes are kept in a separate file
+ * for ease of use in third party programs.
+ */
+#include "libpq/protocol.h"
+
typedef struct
{
struct sockaddr_storage addr;
@@ -112,23 +118,6 @@ typedef uint32 PacketLen;
#define MAX_STARTUP_PACKET_LENGTH 10000
-/* These are the authentication request codes sent by the backend. */
-
-#define AUTH_REQ_OK 0 /* User is authenticated */
-#define AUTH_REQ_KRB4 1 /* Kerberos V4. Not supported any more. */
-#define AUTH_REQ_KRB5 2 /* Kerberos V5. Not supported any more. */
-#define AUTH_REQ_PASSWORD 3 /* Password */
-#define AUTH_REQ_CRYPT 4 /* crypt password. Not supported any more. */
-#define AUTH_REQ_MD5 5 /* md5 password */
-/* 6 is available. It was used for SCM creds, not supported any more. */
-#define AUTH_REQ_GSS 7 /* GSSAPI without wrap() */
-#define AUTH_REQ_GSS_CONT 8 /* Continue GSS exchanges */
-#define AUTH_REQ_SSPI 9 /* SSPI negotiate without wrap() */
-#define AUTH_REQ_SASL 10 /* Begin SASL authentication */
-#define AUTH_REQ_SASL_CONT 11 /* Continue SASL authentication */
-#define AUTH_REQ_SASL_FIN 12 /* Final SASL message */
-#define AUTH_REQ_MAX AUTH_REQ_SASL_FIN /* maximum AUTH_REQ_* value */
-
typedef uint32 AuthRequest;
diff --git a/src/include/libpq/protocol.h b/src/include/libpq/protocol.h
new file mode 100644
index 0000000000..cc46f4b586
--- /dev/null
+++ b/src/include/libpq/protocol.h
@@ -0,0 +1,85 @@
+/*-------------------------------------------------------------------------
+ *
+ * protocol.h
+ * Definitions of the request/response codes for the wire protocol.
+ *
+ *
+ * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/include/libpq/protocol.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef PROTOCOL_H
+#define PROTOCOL_H
+
+/* These are the request codes sent by the frontend. */
+
+#define PqMsg_Bind 'B'
+#define PqMsg_Close 'C'
+#define PqMsg_Describe 'D'
+#define PqMsg_Execute 'E'
+#define PqMsg_FunctionCall 'F'
+#define PqMsg_Flush 'H'
+#define PqMsg_Parse 'P'
+#define PqMsg_Query 'Q'
+#define PqMsg_Sync 'S'
+#define PqMsg_Terminate 'X'
+#define PqMsg_CopyFail 'f'
+#define PqMsg_GSSResponse 'p'
+#define PqMsg_PasswordMessage 'p'
+#define PqMsg_SASLInitialResponse 'p'
+#define PqMsg_SASLResponse 'p'
+
+
+/* These are the response codes sent by the backend. */
+
+#define PqMsg_ParseComplete '1'
+#define PqMsg_BindComplete '2'
+#define PqMsg_CloseComplete '3'
+#define PqMsg_NotificationResponse 'A'
+#define PqMsg_CommandComplete 'C'
+#define PqMsg_DataRow 'D'
+#define PqMsg_ErrorResponse 'E'
+#define PqMsg_CopyInResponse 'G'
+#define PqMsg_CopyOutResponse 'H'
+#define PqMsg_EmptyQueryResponse 'I'
+#define PqMsg_BackendKeyData 'K'
+#define PqMsg_NoticeResponse 'N'
+#define PqMsg_AuthenticationRequest 'R'
+#define PqMsg_ParameterStatus 'S'
+#define PqMsg_RowDescription 'T'
+#define PqMsg_FunctionCallResponse 'V'
+#define PqMsg_CopyBothResponse 'W'
+#define PqMsg_ReadyForQuery 'Z'
+#define PqMsg_NoData 'n'
+#define PqMsg_PortalSuspended 's'
+#define PqMsg_ParameterDescription 't'
+#define PqMsg_NegotiateProtocolVersion 'v'
+
+
+/* These are the codes sent by both the frontend and backend. */
+
+#define PqMsg_CopyDone 'c'
+#define PqMsg_CopyData 'd'
+
+
+/* These are the authentication request codes sent by the backend. */
+
+#define AUTH_REQ_OK 0 /* User is authenticated */
+#define AUTH_REQ_KRB4 1 /* Kerberos V4. Not supported any more. */
+#define AUTH_REQ_KRB5 2 /* Kerberos V5. Not supported any more. */
+#define AUTH_REQ_PASSWORD 3 /* Password */
+#define AUTH_REQ_CRYPT 4 /* crypt password. Not supported any more. */
+#define AUTH_REQ_MD5 5 /* md5 password */
+/* 6 is available. It was used for SCM creds, not supported any more. */
+#define AUTH_REQ_GSS 7 /* GSSAPI without wrap() */
+#define AUTH_REQ_GSS_CONT 8 /* Continue GSS exchanges */
+#define AUTH_REQ_SSPI 9 /* SSPI negotiate without wrap() */
+#define AUTH_REQ_SASL 10 /* Begin SASL authentication */
+#define AUTH_REQ_SASL_CONT 11 /* Continue SASL authentication */
+#define AUTH_REQ_SASL_FIN 12 /* Final SASL message */
+#define AUTH_REQ_MAX AUTH_REQ_SASL_FIN /* maximum AUTH_REQ_* value */
+
+#endif /* PROTOCOL_H */
diff --git a/src/include/meson.build b/src/include/meson.build
index d7e1ecd4c9..d50897c9fd 100644
--- a/src/include/meson.build
+++ b/src/include/meson.build
@@ -94,6 +94,7 @@ install_headers(
install_headers(
'libpq/pqcomm.h',
+ 'libpq/protocol.h',
install_dir: dir_include_internal / 'libpq',
)
diff --git a/src/interfaces/libpq/fe-auth.c b/src/interfaces/libpq/fe-auth.c
index 887ca5e9e1..912aa14821 100644
--- a/src/interfaces/libpq/fe-auth.c
+++ b/src/interfaces/libpq/fe-auth.c
@@ -586,7 +586,7 @@ pg_SASL_init(PGconn *conn, int payloadlen)
/*
* Build a SASLInitialResponse message, and send it.
*/
- if (pqPutMsgStart('p', conn))
+ if (pqPutMsgStart(PqMsg_SASLInitialResponse, conn))
goto error;
if (pqPuts(selected_mechanism, conn))
goto error;
diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c
index 837c5321aa..bf83a9b569 100644
--- a/src/interfaces/libpq/fe-connect.c
+++ b/src/interfaces/libpq/fe-connect.c
@@ -3591,7 +3591,9 @@ keep_going: /* We will come back to here until there is
* Anything else probably means it's not Postgres on the other
* end at all.
*/
- if (!(beresp == 'R' || beresp == 'v' || beresp == 'E'))
+ if (beresp != PqMsg_AuthenticationRequest &&
+ beresp != PqMsg_ErrorResponse &&
+ beresp != PqMsg_NegotiateProtocolVersion)
{
libpq_append_conn_error(conn, "expected authentication request from server, but received %c",
beresp);
@@ -3618,19 +3620,22 @@ keep_going: /* We will come back to here until there is
* version 14, the server also used the old protocol for
* errors that happened before processing the startup packet.)
*/
- if (beresp == 'R' && (msgLength < 8 || msgLength > 2000))
+ if (beresp == PqMsg_AuthenticationRequest &&
+ (msgLength < 8 || msgLength > 2000))
{
libpq_append_conn_error(conn, "received invalid authentication request");
goto error_return;
}
- if (beresp == 'v' && (msgLength < 8 || msgLength > 2000))
+ if (beresp == PqMsg_NegotiateProtocolVersion &&
+ (msgLength < 8 || msgLength > 2000))
{
libpq_append_conn_error(conn, "received invalid protocol negotiation message");
goto error_return;
}
#define MAX_ERRLEN 30000
- if (beresp == 'E' && (msgLength < 8 || msgLength > MAX_ERRLEN))
+ if (beresp == PqMsg_ErrorResponse &&
+ (msgLength < 8 || msgLength > MAX_ERRLEN))
{
/* Handle error from a pre-3.0 server */
conn->inCursor = conn->inStart + 1; /* reread data */
@@ -3693,7 +3698,7 @@ keep_going: /* We will come back to here until there is
}
/* Handle errors. */
- if (beresp == 'E')
+ if (beresp == PqMsg_ErrorResponse)
{
if (pqGetErrorNotice3(conn, true))
{
@@ -3770,7 +3775,7 @@ keep_going: /* We will come back to here until there is
goto error_return;
}
- else if (beresp == 'v')
+ else if (beresp == PqMsg_NegotiateProtocolVersion)
{
if (pqGetNegotiateProtocolVersion3(conn))
{
@@ -4540,7 +4545,7 @@ sendTerminateConn(PGconn *conn)
* Try to send "close connection" message to backend. Ignore any
* error.
*/
- pqPutMsgStart('X', conn);
+ pqPutMsgStart(PqMsg_Terminate, conn);
pqPutMsgEnd(conn);
(void) pqFlush(conn);
}
diff --git a/src/interfaces/libpq/fe-exec.c b/src/interfaces/libpq/fe-exec.c
index a868284ff8..974d462d4b 100644
--- a/src/interfaces/libpq/fe-exec.c
+++ b/src/interfaces/libpq/fe-exec.c
@@ -1458,7 +1458,7 @@ PQsendQueryInternal(PGconn *conn, const char *query, bool newQuery)
/* Send the query message(s) */
/* construct the outgoing Query message */
- if (pqPutMsgStart('Q', conn) < 0 ||
+ if (pqPutMsgStart(PqMsg_Query, conn) < 0 ||
pqPuts(query, conn) < 0 ||
pqPutMsgEnd(conn) < 0)
{
@@ -1571,7 +1571,7 @@ PQsendPrepare(PGconn *conn,
return 0; /* error msg already set */
/* construct the Parse message */
- if (pqPutMsgStart('P', conn) < 0 ||
+ if (pqPutMsgStart(PqMsg_Parse, conn) < 0 ||
pqPuts(stmtName, conn) < 0 ||
pqPuts(query, conn) < 0)
goto sendFailed;
@@ -1599,7 +1599,7 @@ PQsendPrepare(PGconn *conn,
/* Add a Sync, unless in pipeline mode. */
if (conn->pipelineStatus == PQ_PIPELINE_OFF)
{
- if (pqPutMsgStart('S', conn) < 0 ||
+ if (pqPutMsgStart(PqMsg_Sync, conn) < 0 ||
pqPutMsgEnd(conn) < 0)
goto sendFailed;
}
@@ -1784,7 +1784,7 @@ PQsendQueryGuts(PGconn *conn,
if (command)
{
/* construct the Parse message */
- if (pqPutMsgStart('P', conn) < 0 ||
+ if (pqPutMsgStart(PqMsg_Parse, conn) < 0 ||
pqPuts(stmtName, conn) < 0 ||
pqPuts(command, conn) < 0)
goto sendFailed;
@@ -1808,7 +1808,7 @@ PQsendQueryGuts(PGconn *conn,
}
/* Construct the Bind message */
- if (pqPutMsgStart('B', conn) < 0 ||
+ if (pqPutMsgStart(PqMsg_Bind, conn) < 0 ||
pqPuts("", conn) < 0 ||
pqPuts(stmtName, conn) < 0)
goto sendFailed;
@@ -1874,14 +1874,14 @@ PQsendQueryGuts(PGconn *conn,
goto sendFailed;
/* construct the Describe Portal message */
- if (pqPutMsgStart('D', conn) < 0 ||
+ if (pqPutMsgStart(PqMsg_Describe, conn) < 0 ||
pqPutc('P', conn) < 0 ||
pqPuts("", conn) < 0 ||
pqPutMsgEnd(conn) < 0)
goto sendFailed;
/* construct the Execute message */
- if (pqPutMsgStart('E', conn) < 0 ||
+ if (pqPutMsgStart(PqMsg_Execute, conn) < 0 ||
pqPuts("", conn) < 0 ||
pqPutInt(0, 4, conn) < 0 ||
pqPutMsgEnd(conn) < 0)
@@ -1890,7 +1890,7 @@ PQsendQueryGuts(PGconn *conn,
/* construct the Sync message if not in pipeline mode */
if (conn->pipelineStatus == PQ_PIPELINE_OFF)
{
- if (pqPutMsgStart('S', conn) < 0 ||
+ if (pqPutMsgStart(PqMsg_Sync, conn) < 0 ||
pqPutMsgEnd(conn) < 0)
goto sendFailed;
}
@@ -2422,7 +2422,7 @@ PQdescribePrepared(PGconn *conn, const char *stmt)
{
if (!PQexecStart(conn))
return NULL;
- if (!PQsendTypedCommand(conn, 'D', 'S', stmt))
+ if (!PQsendTypedCommand(conn, PqMsg_Describe, 'S', stmt))
return NULL;
return PQexecFinish(conn);
}
@@ -2441,7 +2441,7 @@ PQdescribePortal(PGconn *conn, const char *portal)
{
if (!PQexecStart(conn))
return NULL;
- if (!PQsendTypedCommand(conn, 'D', 'P', portal))
+ if (!PQsendTypedCommand(conn, PqMsg_Describe, 'P', portal))
return NULL;
return PQexecFinish(conn);
}
@@ -2456,7 +2456,7 @@ PQdescribePortal(PGconn *conn, const char *portal)
int
PQsendDescribePrepared(PGconn *conn, const char *stmt)
{
- return PQsendTypedCommand(conn, 'D', 'S', stmt);
+ return PQsendTypedCommand(conn, PqMsg_Describe, 'S', stmt);
}
/*
@@ -2469,7 +2469,7 @@ PQsendDescribePrepared(PGconn *conn, const char *stmt)
int
PQsendDescribePortal(PGconn *conn, const char *portal)
{
- return PQsendTypedCommand(conn, 'D', 'P', portal);
+ return PQsendTypedCommand(conn, PqMsg_Describe, 'P', portal);
}
/*
@@ -2488,7 +2488,7 @@ PQclosePrepared(PGconn *conn, const char *stmt)
{
if (!PQexecStart(conn))
return NULL;
- if (!PQsendTypedCommand(conn, 'C', 'S', stmt))
+ if (!PQsendTypedCommand(conn, PqMsg_Close, 'S', stmt))
return NULL;
return PQexecFinish(conn);
}
@@ -2506,7 +2506,7 @@ PQclosePortal(PGconn *conn, const char *portal)
{
if (!PQexecStart(conn))
return NULL;
- if (!PQsendTypedCommand(conn, 'C', 'P', portal))
+ if (!PQsendTypedCommand(conn, PqMsg_Close, 'P', portal))
return NULL;
return PQexecFinish(conn);
}
@@ -2521,7 +2521,7 @@ PQclosePortal(PGconn *conn, const char *portal)
int
PQsendClosePrepared(PGconn *conn, const char *stmt)
{
- return PQsendTypedCommand(conn, 'C', 'S', stmt);
+ return PQsendTypedCommand(conn, PqMsg_Close, 'S', stmt);
}
/*
@@ -2534,7 +2534,7 @@ PQsendClosePrepared(PGconn *conn, const char *stmt)
int
PQsendClosePortal(PGconn *conn, const char *portal)
{
- return PQsendTypedCommand(conn, 'C', 'P', portal);
+ return PQsendTypedCommand(conn, PqMsg_Close, 'P', portal);
}
/*
@@ -2542,8 +2542,8 @@ PQsendClosePortal(PGconn *conn, const char *portal)
* Common code to send a Describe or Close command
*
* Available options for "command" are
- * 'C' for Close; or
- * 'D' for Describe.
+ * PqMsg_Close for Close; or
+ * PqMsg_Describe for Describe.
*
* Available options for "type" are
* 'S' to run a command on a prepared statement; or
@@ -2577,17 +2577,17 @@ PQsendTypedCommand(PGconn *conn, char command, char type, const char *target)
/* construct the Sync message */
if (conn->pipelineStatus == PQ_PIPELINE_OFF)
{
- if (pqPutMsgStart('S', conn) < 0 ||
+ if (pqPutMsgStart(PqMsg_Sync, conn) < 0 ||
pqPutMsgEnd(conn) < 0)
goto sendFailed;
}
/* remember if we are doing a Close or a Describe */
- if (command == 'C')
+ if (command == PqMsg_Close)
{
entry->queryclass = PGQUERY_CLOSE;
}
- else if (command == 'D')
+ else if (command == PqMsg_Describe)
{
entry->queryclass = PGQUERY_DESCRIBE;
}
@@ -2696,7 +2696,7 @@ PQputCopyData(PGconn *conn, const char *buffer, int nbytes)
return pqIsnonblocking(conn) ? 0 : -1;
}
/* Send the data (too simple to delegate to fe-protocol files) */
- if (pqPutMsgStart('d', conn) < 0 ||
+ if (pqPutMsgStart(PqMsg_CopyData, conn) < 0 ||
pqPutnchar(buffer, nbytes, conn) < 0 ||
pqPutMsgEnd(conn) < 0)
return -1;
@@ -2731,7 +2731,7 @@ PQputCopyEnd(PGconn *conn, const char *errormsg)
if (errormsg)
{
/* Send COPY FAIL */
- if (pqPutMsgStart('f', conn) < 0 ||
+ if (pqPutMsgStart(PqMsg_CopyFail, conn) < 0 ||
pqPuts(errormsg, conn) < 0 ||
pqPutMsgEnd(conn) < 0)
return -1;
@@ -2739,7 +2739,7 @@ PQputCopyEnd(PGconn *conn, const char *errormsg)
else
{
/* Send COPY DONE */
- if (pqPutMsgStart('c', conn) < 0 ||
+ if (pqPutMsgStart(PqMsg_CopyDone, conn) < 0 ||
pqPutMsgEnd(conn) < 0)
return -1;
}
@@ -2751,7 +2751,7 @@ PQputCopyEnd(PGconn *conn, const char *errormsg)
if (conn->cmd_queue_head &&
conn->cmd_queue_head->queryclass != PGQUERY_SIMPLE)
{
- if (pqPutMsgStart('S', conn) < 0 ||
+ if (pqPutMsgStart(PqMsg_Sync, conn) < 0 ||
pqPutMsgEnd(conn) < 0)
return -1;
}
@@ -3263,7 +3263,7 @@ PQpipelineSync(PGconn *conn)
entry->query = NULL;
/* construct the Sync message */
- if (pqPutMsgStart('S', conn) < 0 ||
+ if (pqPutMsgStart(PqMsg_Sync, conn) < 0 ||
pqPutMsgEnd(conn) < 0)
goto sendFailed;
@@ -3311,7 +3311,7 @@ PQsendFlushRequest(PGconn *conn)
return 0;
}
- if (pqPutMsgStart('H', conn) < 0 ||
+ if (pqPutMsgStart(PqMsg_Flush, conn) < 0 ||
pqPutMsgEnd(conn) < 0)
{
return 0;
diff --git a/src/interfaces/libpq/fe-protocol3.c b/src/interfaces/libpq/fe-protocol3.c
index 7bc6355d17..5613c56b14 100644
--- a/src/interfaces/libpq/fe-protocol3.c
+++ b/src/interfaces/libpq/fe-protocol3.c
@@ -34,8 +34,13 @@
* than a couple of kilobytes).
*/
#define VALID_LONG_MESSAGE_TYPE(id) \
- ((id) == 'T' || (id) == 'D' || (id) == 'd' || (id) == 'V' || \
- (id) == 'E' || (id) == 'N' || (id) == 'A')
+ ((id) == PqMsg_CopyData || \
+ (id) == PqMsg_DataRow || \
+ (id) == PqMsg_ErrorResponse || \
+ (id) == PqMsg_FunctionCallResponse || \
+ (id) == PqMsg_NoticeResponse || \
+ (id) == PqMsg_NotificationResponse || \
+ (id) == PqMsg_RowDescription)
static void handleSyncLoss(PGconn *conn, char id, int msgLength);
@@ -140,12 +145,12 @@ pqParseInput3(PGconn *conn)
* from config file due to SIGHUP), but otherwise we hold off until
* BUSY state.
*/
- if (id == 'A')
+ if (id == PqMsg_NotificationResponse)
{
if (getNotify(conn))
return;
}
- else if (id == 'N')
+ else if (id == PqMsg_NoticeResponse)
{
if (pqGetErrorNotice3(conn, false))
return;
@@ -165,12 +170,12 @@ pqParseInput3(PGconn *conn)
* it is about to close the connection, so we don't want to just
* discard it...)
*/
- if (id == 'E')
+ if (id == PqMsg_ErrorResponse)
{
if (pqGetErrorNotice3(conn, false /* treat as notice */ ))
return;
}
- else if (id == 'S')
+ else if (id == PqMsg_ParameterStatus)
{
if (getParameterStatus(conn))
return;
@@ -192,7 +197,7 @@ pqParseInput3(PGconn *conn)
*/
switch (id)
{
- case 'C': /* command complete */
+ case PqMsg_CommandComplete:
if (pqGets(&conn->workBuffer, conn))
return;
if (!pgHavePendingResult(conn))
@@ -210,13 +215,12 @@ pqParseInput3(PGconn *conn)
CMDSTATUS_LEN);
conn->asyncStatus = PGASYNC_READY;
break;
- case 'E': /* error return */
+ case PqMsg_ErrorResponse:
if (pqGetErrorNotice3(conn, true))
return;
conn->asyncStatus = PGASYNC_READY;
break;
- case 'Z': /* sync response, backend is ready for new
- * query */
+ case PqMsg_ReadyForQuery:
if (getReadyForQuery(conn))
return;
if (conn->pipelineStatus != PQ_PIPELINE_OFF)
@@ -246,7 +250,7 @@ pqParseInput3(PGconn *conn)
conn->asyncStatus = PGASYNC_IDLE;
}
break;
- case 'I': /* empty query */
+ case PqMsg_EmptyQueryResponse:
if (!pgHavePendingResult(conn))
{
conn->result = PQmakeEmptyPGresult(conn,
@@ -259,7 +263,7 @@ pqParseInput3(PGconn *conn)
}
conn->asyncStatus = PGASYNC_READY;
break;
- case '1': /* Parse Complete */
+ case PqMsg_ParseComplete:
/* If we're doing PQprepare, we're done; else ignore */
if (conn->cmd_queue_head &&
conn->cmd_queue_head->queryclass == PGQUERY_PREPARE)
@@ -277,10 +281,10 @@ pqParseInput3(PGconn *conn)
conn->asyncStatus = PGASYNC_READY;
}
break;
- case '2': /* Bind Complete */
+ case PqMsg_BindComplete:
/* Nothing to do for this message type */
break;
- case '3': /* Close Complete */
+ case PqMsg_CloseComplete:
/* If we're doing PQsendClose, we're done; else ignore */
if (conn->cmd_queue_head &&
conn->cmd_queue_head->queryclass == PGQUERY_CLOSE)
@@ -298,11 +302,11 @@ pqParseInput3(PGconn *conn)
conn->asyncStatus = PGASYNC_READY;
}
break;
- case 'S': /* parameter status */
+ case PqMsg_ParameterStatus:
if (getParameterStatus(conn))
return;
break;
- case 'K': /* secret key data from the backend */
+ case PqMsg_BackendKeyData:
/*
* This is expected only during backend startup, but it's
@@ -314,7 +318,7 @@ pqParseInput3(PGconn *conn)
if (pqGetInt(&(conn->be_key), 4, conn))
return;
break;
- case 'T': /* Row Description */
+ case PqMsg_RowDescription:
if (conn->error_result ||
(conn->result != NULL &&
conn->result->resultStatus == PGRES_FATAL_ERROR))
@@ -346,7 +350,7 @@ pqParseInput3(PGconn *conn)
return;
}
break;
- case 'n': /* No Data */
+ case PqMsg_NoData:
/*
* NoData indicates that we will not be seeing a
@@ -374,11 +378,11 @@ pqParseInput3(PGconn *conn)
conn->asyncStatus = PGASYNC_READY;
}
break;
- case 't': /* Parameter Description */
+ case PqMsg_ParameterDescription:
if (getParamDescriptions(conn, msgLength))
return;
break;
- case 'D': /* Data Row */
+ case PqMsg_DataRow:
if (conn->result != NULL &&
conn->result->resultStatus == PGRES_TUPLES_OK)
{
@@ -405,24 +409,24 @@ pqParseInput3(PGconn *conn)
conn->inCursor += msgLength;
}
break;
- case 'G': /* Start Copy In */
+ case PqMsg_CopyInResponse:
if (getCopyStart(conn, PGRES_COPY_IN))
return;
conn->asyncStatus = PGASYNC_COPY_IN;
break;
- case 'H': /* Start Copy Out */
+ case PqMsg_CopyOutResponse:
if (getCopyStart(conn, PGRES_COPY_OUT))
return;
conn->asyncStatus = PGASYNC_COPY_OUT;
conn->copy_already_done = 0;
break;
- case 'W': /* Start Copy Both */
+ case PqMsg_CopyBothResponse:
if (getCopyStart(conn, PGRES_COPY_BOTH))
return;
conn->asyncStatus = PGASYNC_COPY_BOTH;
conn->copy_already_done = 0;
break;
- case 'd': /* Copy Data */
+ case PqMsg_CopyData:
/*
* If we see Copy Data, just silently drop it. This would
@@ -431,7 +435,7 @@ pqParseInput3(PGconn *conn)
*/
conn->inCursor += msgLength;
break;
- case 'c': /* Copy Done */
+ case PqMsg_CopyDone:
/*
* If we see Copy Done, just silently drop it. This is
@@ -1692,21 +1696,21 @@ getCopyDataMessage(PGconn *conn)
*/
switch (id)
{
- case 'A': /* NOTIFY */
+ case PqMsg_NotificationResponse:
if (getNotify(conn))
return 0;
break;
- case 'N': /* NOTICE */
+ case PqMsg_NoticeResponse:
if (pqGetErrorNotice3(conn, false))
return 0;
break;
- case 'S': /* ParameterStatus */
+ case PqMsg_ParameterStatus:
if (getParameterStatus(conn))
return 0;
break;
- case 'd': /* Copy Data, pass it back to caller */
+ case PqMsg_CopyData:
return msgLength;
- case 'c':
+ case PqMsg_CopyDone:
/*
* If this is a CopyDone message, exit COPY_OUT mode and let
@@ -1929,7 +1933,7 @@ pqEndcopy3(PGconn *conn)
if (conn->asyncStatus == PGASYNC_COPY_IN ||
conn->asyncStatus == PGASYNC_COPY_BOTH)
{
- if (pqPutMsgStart('c', conn) < 0 ||
+ if (pqPutMsgStart(PqMsg_CopyDone, conn) < 0 ||
pqPutMsgEnd(conn) < 0)
return 1;
@@ -1940,7 +1944,7 @@ pqEndcopy3(PGconn *conn)
if (conn->cmd_queue_head &&
conn->cmd_queue_head->queryclass != PGQUERY_SIMPLE)
{
- if (pqPutMsgStart('S', conn) < 0 ||
+ if (pqPutMsgStart(PqMsg_Sync, conn) < 0 ||
pqPutMsgEnd(conn) < 0)
return 1;
}
@@ -2023,7 +2027,7 @@ pqFunctionCall3(PGconn *conn, Oid fnid,
/* PQfn already validated connection state */
- if (pqPutMsgStart('F', conn) < 0 || /* function call msg */
+ if (pqPutMsgStart(PqMsg_FunctionCall, conn) < 0 ||
pqPutInt(fnid, 4, conn) < 0 || /* function id */
pqPutInt(1, 2, conn) < 0 || /* # of format codes */
pqPutInt(1, 2, conn) < 0 || /* format code: BINARY */
diff --git a/src/interfaces/libpq/fe-trace.c b/src/interfaces/libpq/fe-trace.c
index 402784f40e..b18e3deab6 100644
--- a/src/interfaces/libpq/fe-trace.c
+++ b/src/interfaces/libpq/fe-trace.c
@@ -562,110 +562,120 @@ pqTraceOutputMessage(PGconn *conn, const char *message, bool toServer)
switch (id)
{
- case '1':
+ case PqMsg_ParseComplete:
fprintf(conn->Pfdebug, "ParseComplete");
/* No message content */
break;
- case '2':
+ case PqMsg_BindComplete:
fprintf(conn->Pfdebug, "BindComplete");
/* No message content */
break;
- case '3':
+ case PqMsg_CloseComplete:
fprintf(conn->Pfdebug, "CloseComplete");
/* No message content */
break;
- case 'A': /* Notification Response */
+ case PqMsg_NotificationResponse:
pqTraceOutputA(conn->Pfdebug, message, &logCursor, regress);
break;
- case 'B': /* Bind */
+ case PqMsg_Bind:
pqTraceOutputB(conn->Pfdebug, message, &logCursor);
break;
- case 'c':
+ case PqMsg_CopyDone:
fprintf(conn->Pfdebug, "CopyDone");
/* No message content */
break;
- case 'C': /* Close(F) or Command Complete(B) */
+ case PqMsg_CommandComplete:
+ /* Close(F) and CommandComplete(B) use the same identifier. */
+ Assert(PqMsg_Close == PqMsg_CommandComplete);
pqTraceOutputC(conn->Pfdebug, toServer, message, &logCursor);
break;
- case 'd': /* Copy Data */
+ case PqMsg_CopyData:
/* Drop COPY data to reduce the overhead of logging. */
break;
- case 'D': /* Describe(F) or Data Row(B) */
+ case PqMsg_Describe:
+ /* Describe(F) and DataRow(B) use the same identifier. */
+ Assert(PqMsg_Describe == PqMsg_DataRow);
pqTraceOutputD(conn->Pfdebug, toServer, message, &logCursor);
break;
- case 'E': /* Execute(F) or Error Response(B) */
+ case PqMsg_Execute:
+ /* Execute(F) and ErrorResponse(B) use the same identifier. */
+ Assert(PqMsg_Execute == PqMsg_ErrorResponse);
pqTraceOutputE(conn->Pfdebug, toServer, message, &logCursor,
regress);
break;
- case 'f': /* Copy Fail */
+ case PqMsg_CopyFail:
pqTraceOutputf(conn->Pfdebug, message, &logCursor);
break;
- case 'F': /* Function Call */
+ case PqMsg_FunctionCall:
pqTraceOutputF(conn->Pfdebug, message, &logCursor, regress);
break;
- case 'G': /* Start Copy In */
+ case PqMsg_CopyInResponse:
pqTraceOutputG(conn->Pfdebug, message, &logCursor);
break;
- case 'H': /* Flush(F) or Start Copy Out(B) */
+ case PqMsg_Flush:
+ /* Flush(F) and CopyOutResponse(B) use the same identifier */
+ Assert(PqMsg_CopyOutResponse == PqMsg_Flush);
if (!toServer)
pqTraceOutputH(conn->Pfdebug, message, &logCursor);
else
fprintf(conn->Pfdebug, "Flush"); /* no message content */
break;
- case 'I':
+ case PqMsg_EmptyQueryResponse:
fprintf(conn->Pfdebug, "EmptyQueryResponse");
/* No message content */
break;
- case 'K': /* secret key data from the backend */
+ case PqMsg_BackendKeyData:
pqTraceOutputK(conn->Pfdebug, message, &logCursor, regress);
break;
- case 'n':
+ case PqMsg_NoData:
fprintf(conn->Pfdebug, "NoData");
/* No message content */
break;
- case 'N':
+ case PqMsg_NoticeResponse:
pqTraceOutputNR(conn->Pfdebug, "NoticeResponse", message,
&logCursor, regress);
break;
- case 'P': /* Parse */
+ case PqMsg_Parse:
pqTraceOutputP(conn->Pfdebug, message, &logCursor, regress);
break;
- case 'Q': /* Query */
+ case PqMsg_Query:
pqTraceOutputQ(conn->Pfdebug, message, &logCursor);
break;
- case 'R': /* Authentication */
+ case PqMsg_AuthenticationRequest:
pqTraceOutputR(conn->Pfdebug, message, &logCursor);
break;
- case 's':
+ case PqMsg_PortalSuspended:
fprintf(conn->Pfdebug, "PortalSuspended");
/* No message content */
break;
- case 'S': /* Parameter Status(B) or Sync(F) */
+ case PqMsg_Sync:
+ /* Parameter Status(B) and Sync(F) use the same identifier */
+ Assert(PqMsg_ParameterStatus == PqMsg_Sync);
if (!toServer)
pqTraceOutputS(conn->Pfdebug, message, &logCursor);
else
fprintf(conn->Pfdebug, "Sync"); /* no message content */
break;
- case 't': /* Parameter Description */
+ case PqMsg_ParameterDescription:
pqTraceOutputt(conn->Pfdebug, message, &logCursor, regress);
break;
- case 'T': /* Row Description */
+ case PqMsg_RowDescription:
pqTraceOutputT(conn->Pfdebug, message, &logCursor, regress);
break;
- case 'v': /* Negotiate Protocol Version */
+ case PqMsg_NegotiateProtocolVersion:
pqTraceOutputv(conn->Pfdebug, message, &logCursor);
break;
- case 'V': /* Function Call response */
+ case PqMsg_FunctionCallResponse:
pqTraceOutputV(conn->Pfdebug, message, &logCursor);
break;
- case 'W': /* Start Copy Both */
+ case PqMsg_CopyBothResponse:
pqTraceOutputW(conn->Pfdebug, message, &logCursor, length);
break;
- case 'X':
+ case PqMsg_Terminate:
fprintf(conn->Pfdebug, "Terminate");
/* No message content */
break;
- case 'Z': /* Ready For Query */
+ case PqMsg_ReadyForQuery:
pqTraceOutputZ(conn->Pfdebug, message, &logCursor);
break;
default:
diff --git a/src/tools/msvc/Install.pm b/src/tools/msvc/Install.pm
index 05548d7c0a..b6dd2c3bba 100644
--- a/src/tools/msvc/Install.pm
+++ b/src/tools/msvc/Install.pm
@@ -614,6 +614,8 @@ sub CopyIncludeFiles
'src/include/', 'c.h', 'port.h', 'postgres_fe.h');
lcopy('src/include/libpq/pqcomm.h', $target . '/include/internal/libpq/')
|| croak 'Could not copy pqcomm.h';
+ lcopy('src/include/libpq/protocol.h', $target . '/include/internal/libpq/')
+ || croak 'Could not copy protocol.h';
CopyFiles(
'Server headers',
--
2.25.1
--W/nzBZO5zC0uMSeA--
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
@ 2023-12-20 21:48 Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
0 siblings, 1 reply; 46+ messages in thread
From: Jacob Burroughs @ 2023-12-20 21:48 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: [email protected]
> I'm having difficulty understanding the details of this handshaking
> algorithm from this description. It seems good that the handshake
> proceeds in each direction somewhat separately from the other, but I
> don't quite understand how the whole thing fits together. If the
> client tells the server that 'none,gzip' is supported, and I elect to
> start using gzip, how does the client know that I picked gzip rather
> than none? Are the compressed packets self-identifying?
I agree I could have spelled this out more clearly. I forgot to
mention that I added a byte to the CompressedMessage message type that
specifies the chosen algorithm. So if the server receives
'none,gzip', it can either keep sending uncompressed regular messages,
or it can compress them in CompressedMessage packets which now look
like "z{len}{format}{data}" (format just being a member of the
pg_compress_algorithm enum, so `1` in the case of gzip). Overall the
intention is that both the client and the server can just start
sending CompressedMessages once they receive the list of ones other
party supports without any negotiation or agreement needed and without
an extra message type to first specify the compression algorithm. (One
byte per message seemed to me like a reasonable overhead for the
simplicity, but it wouldn't be hard to bring back SetCompressionMethod
if we prefer.)
> It's also slightly odd to me that the same parameter seems to specify
> both what we want to send, and what we're able to receive. I'm not
> really sure we should have separate parameters for those things, but I
> don't quite understand how this works without it. The "none" thing
> seems like a bit of a hack. It lets you say "I'd like to receive
> compressed data but send uncompressed data" ... but what about the
> reverse? How do you say "don't bother compressing what you receive
> from the server, but please lz4 everything you send to the server"? Or
> how do you say "use zstd from server to client, but lz4 from client to
> server"? It seems like you can't really say that kind of thing.
When I came up with the protocol I was imagining that basically both
server admins and clients might want a decent bit more control over
the compression they do rather than the decompression they do, since
compression is generally much more computationally expensive than
decompression. Now that you point it out though, I don't think that
actually makes that much sense.
> What if we had, on the server side, a GUC saying what compression to
> accept and a GUC saying what compression to be willing to do? And then
> let the client request whatever it wants for each direction.
Here's two proposals:
Option 1:
GUCs:
libpq_compression (default "off")
libpq_decompression (default "auto", which is defined to be equal to
libpq_compression)
Connection parameters:
compression (default "off")
decompression (default "auto", which is defined to be equal to compression)
I think we would only send the decompression fields over the wire to
the other side, to be used to filter for the first chosen compression
field. We would send the `_pq_.libpq_decompression` protocol
extension even if only compression was enabled and not decompression
so that the server knows to enable compression processing for the
connection (I think this would be the only place we would still use
`none`, and not as part of a list in this case.) I think we also
would want to add libpq functions to allow a client to check the
last-used compression algorithm in each direction for any
monitoring/inspection purposes (actually that's probably a good idea
regardless, so a client application that cares doesn't need to/try to
implement the intersection and assumption around choosing the first
algorithm in common). Also I'm open to better names than "auto", I
just would like it to avoid unnecessary verbosity for the common case
of "I just want to enable bidirectional compression with whatever
algorithms are available with default parameters".
Option 2:
This one is even cleaner in the common case but a bit worse in the
uncommon case: just use one parameter and have
compression/decompression enabling be part of the compression detail
(e.g. "libpq_compression='gzip:no_decompress;lz4:level=2,no_compress;zstd'"
or something like that, in which case the "none,gzip" case would
become "'libpq_compression=gzip:no_compress'"). See
https://www.postgresql.org/docs/current/app-pgbasebackup.html ,
specifically the `--compress` flag, for how specifying compression
algorithms and details works.
I'm actually not sure which of the two I prefer; opinions are welcome :)
Thanks,
Jacob
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
@ 2023-12-21 00:30 ` Jelte Fennema-Nio <[email protected]>
2023-12-29 10:02 ` Re: libpq compression (part 3) Andrey M. Borodin <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
0 siblings, 2 replies; 46+ messages in thread
From: Jelte Fennema-Nio @ 2023-12-21 00:30 UTC (permalink / raw)
To: Jacob Burroughs <[email protected]>; +Cc: Robert Haas <[email protected]>; PostgreSQL Hackers <[email protected]>
Thanks for working on this!
One thing I'm wondering: should it be possible for the client to change the
compression it wants mid-connection? I can think of some scenarios where
that would be useful to connection poolers: if a pooler does plain
forwarding of the compressed messages, then it would need to be able to
disable/enable compression if it wants to multiplex client connections with
different compression settings over the same server connection.
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
@ 2023-12-29 10:02 ` Andrey M. Borodin <[email protected]>
1 sibling, 0 replies; 46+ messages in thread
From: Andrey M. Borodin @ 2023-12-29 10:02 UTC (permalink / raw)
To: Jelte Fennema-Nio <[email protected]>; +Cc: Jacob Burroughs <[email protected]>; Robert Haas <[email protected]>; PostgreSQL Hackers <[email protected]>
> On 21 Dec 2023, at 05:30, Jelte Fennema-Nio <[email protected]> wrote:
>
> One thing I'm wondering: should it be possible for the client to change the compression it wants mid-connection?
This patchset allows sending CompressionMethod message, which allows to set another codec\level picked from the set of negotiated codec sets (during startup).
Best regards, Andrey Borodin.
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
@ 2023-12-31 07:32 ` Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
1 sibling, 1 reply; 46+ messages in thread
From: Jacob Burroughs @ 2023-12-31 07:32 UTC (permalink / raw)
To: Jelte Fennema-Nio <[email protected]>; +Cc: Robert Haas <[email protected]>; PostgreSQL Hackers <[email protected]>
> One thing I'm wondering: should it be possible for the client to change the compression it wants mid-connection? I can think of some scenarios where that would be useful to connection poolers: if a pooler does plain forwarding of the compressed messages, then it would need to be able to disable/enable compression if it wants to multiplex client connections with different compression settings over the same server connection.
I have reworked this patch series to make it easier to extend to
restart compression mid-connection once something in the vein of the
discussion in "Add new protocol message to change GUCs for usage with
future protocol-only GUCs" [1] happens. In particular, I have changed
the `CompressedMessage` protocol message to signal the current
compression algorithm any time the client should restart its streaming
decompressor and otherwise implicitly use whatever compression
algorithm and decompressor was used for previous `CompressedMessage` ,
which future work can leverage to trigger such a restart on update of
the client-supported compression algorithms.
> Option 2:
> This one is even cleaner in the common case but a bit worse in the
> uncommon case: just use one parameter and have
> compression/decompression enabling be part of the compression detail
> (e.g. "libpq_compression='gzip:no_de
> compress;lz4:level=2,no_compress;zstd'"
> or something like that, in which case the "none,gzip" case would
> become "'libpq_compression=gzip:no_compress'"). See
> https://www.postgresql.org/docs/current/app-pgbasebackup.html ,
> specifically the `--compress` flag, for how specifying compression
> algorithms and details works.
I ended up reworking this to use a version of this option in place of
the `none` hackery, but naming the parameters `compress` and
`decompress, so to disable compression but allow decompression you
would specify `libpq_compression=gzip:compress=off`.
Also my windows SSL test failures seem to have resolved themselves
with either these changes or a rebase, so I think things are truly in
a reviewable state now.
Attachments:
[application/octet-stream] v3-0001-Add-IO-stream-abstraction.patch (79.6K, ../../CACzsqT5-7xfbz+Si35TBYHzerNX3XJVzAUH9AewQ+Pp13fYBoQ@mail.gmail.com/2-v3-0001-Add-IO-stream-abstraction.patch)
download | inline diff:
From b0ceb4c404b95b1037e31578037be791206c3d1a Mon Sep 17 00:00:00 2001
From: Jacob Burroughs <[email protected]>
Date: Tue, 12 Dec 2023 16:28:03 -0600
Subject: [PATCH v3 1/5] Add IO stream abstraction
This is a shared abstraction between the frontend and backend that
is designed to draw a clean line between the compoments of postgres
that want to consume "regular" protocol bytes and any layers that
may want to transform them. In this patch, encyryption is simply
moved to use this abstraction rather than callers directly using
secure_read/secure_write (which depending on context might or might
not actually be secure). This is a pre-work change intended to make
compression integrate into frontend and backend comms layers more
cleanly.
This may be a first step towards facilitating greater sharing of
SSL/GSS api code between the frontend and backend down the road,
but for now it the stream processors themselves are completely
independent.
---
src/backend/libpq/be-secure-gssapi.c | 63 ++--
src/backend/libpq/be-secure-openssl.c | 75 +++--
src/backend/libpq/be-secure.c | 216 ------------
src/backend/libpq/pqcomm.c | 245 +++++++++++++-
src/backend/postmaster/postmaster.c | 5 +
src/common/Makefile | 1 +
src/common/io_stream.c | 148 ++++++++
src/common/meson.build | 1 +
src/include/common/io_stream.h | 131 ++++++++
src/include/libpq/libpq-be.h | 23 +-
src/include/libpq/libpq.h | 6 +-
src/interfaces/libpq/fe-connect.c | 343 ++++++++++++++++++-
src/interfaces/libpq/fe-misc.c | 31 +-
src/interfaces/libpq/fe-secure-gssapi.c | 64 ++--
src/interfaces/libpq/fe-secure-openssl.c | 68 +++-
src/interfaces/libpq/fe-secure.c | 411 -----------------------
src/interfaces/libpq/libpq-int.h | 42 +--
17 files changed, 1069 insertions(+), 804 deletions(-)
create mode 100644 src/common/io_stream.c
create mode 100644 src/include/common/io_stream.h
diff --git a/src/backend/libpq/be-secure-gssapi.c b/src/backend/libpq/be-secure-gssapi.c
index 8ed2f65347..597ddbae6e 100644
--- a/src/backend/libpq/be-secure-gssapi.c
+++ b/src/backend/libpq/be-secure-gssapi.c
@@ -74,6 +74,12 @@ static int PqGSSResultNext; /* Next index to read a byte from
static uint32 PqGSSMaxPktSize; /* Maximum size we can encrypt and fit the
* results into our output buffer */
+static ssize_t be_gssapi_read(IoStreamLayer * self, Port *port, void *ptr, size_t len, bool buffered_only);
+static int be_gssapi_write(IoStreamLayer * self, Port *port, void const *ptr, size_t len, size_t *bytes_written);
+IoStreamProcessor be_gssapi_processor = {
+ .read = (io_stream_read_func) be_gssapi_read,
+ .write = (io_stream_write_func) be_gssapi_write
+};
/*
* Attempt to write len bytes of data from ptr to a GSSAPI-encrypted connection.
@@ -91,8 +97,8 @@ static uint32 PqGSSMaxPktSize; /* Maximum size we can encrypt and fit the
* recursion. Instead, use elog(COMMERROR) to log extra info about the
* failure if necessary, and then return an errno indicating connection loss.
*/
-ssize_t
-be_gssapi_write(Port *port, void *ptr, size_t len)
+static int
+be_gssapi_write(IoStreamLayer * self, Port *port, void const *ptr, size_t len, size_t *bytes_written)
{
OM_uint32 major,
minor;
@@ -102,6 +108,8 @@ be_gssapi_write(Port *port, void *ptr, size_t len)
size_t bytes_encrypted;
gss_ctx_id_t gctx = port->gss->ctx;
+ *bytes_written = 0;
+
/*
* When we get a retryable failure, we must not tell the caller we have
* successfully transmitted everything, else it won't retry. For
@@ -148,20 +156,21 @@ be_gssapi_write(Port *port, void *ptr, size_t len)
*/
if (PqGSSSendLength)
{
- ssize_t ret;
+ int retval;
+ size_t count;
ssize_t amount = PqGSSSendLength - PqGSSSendNext;
- ret = secure_raw_write(port, PqGSSSendBuffer + PqGSSSendNext, amount);
- if (ret <= 0)
- return ret;
+ retval = io_stream_next_write(self, PqGSSSendBuffer + PqGSSSendNext, amount, &count);
+ if (retval < 0 || count == 0)
+ return retval;
/*
* Check if this was a partial write, and if so, move forward that
* far in our buffer and try again.
*/
- if (ret < amount)
+ if (count < amount)
{
- PqGSSSendNext += ret;
+ PqGSSSendNext += count;
continue;
}
@@ -242,7 +251,8 @@ be_gssapi_write(Port *port, void *ptr, size_t len)
/* We're reporting all the data as sent, so reset PqGSSSendConsumed. */
PqGSSSendConsumed = 0;
- return bytes_encrypted;
+ *bytes_written = bytes_encrypted;
+ return 0;
}
/*
@@ -258,8 +268,8 @@ be_gssapi_write(Port *port, void *ptr, size_t len)
* We treat fatal errors the same as in be_gssapi_write(), even though the
* argument about infinite recursion doesn't apply here.
*/
-ssize_t
-be_gssapi_read(Port *port, void *ptr, size_t len)
+static ssize_t
+be_gssapi_read(IoStreamLayer * self, Port *port, void *ptr, size_t len, bool buffered_only)
{
OM_uint32 major,
minor;
@@ -269,6 +279,9 @@ be_gssapi_read(Port *port, void *ptr, size_t len)
size_t bytes_returned = 0;
gss_ctx_id_t gctx = port->gss->ctx;
+ if (buffered_only)
+ return false;
+
/*
* The plan here is to read one incoming encrypted packet into
* PqGSSRecvBuffer, decrypt it into PqGSSResultBuffer, and then dole out
@@ -325,10 +338,10 @@ be_gssapi_read(Port *port, void *ptr, size_t len)
/* Collect the length if we haven't already */
if (PqGSSRecvLength < sizeof(uint32))
{
- ret = secure_raw_read(port, PqGSSRecvBuffer + PqGSSRecvLength,
- sizeof(uint32) - PqGSSRecvLength);
+ ret = io_stream_next_read(self, PqGSSRecvBuffer + PqGSSRecvLength,
+ sizeof(uint32) - PqGSSRecvLength, false);
- /* If ret <= 0, secure_raw_read already set the correct errno */
+ /* If ret <= 0, io_stream_next_read already set the correct errno */
if (ret <= 0)
return ret;
@@ -359,8 +372,8 @@ be_gssapi_read(Port *port, void *ptr, size_t len)
* Read as much of the packet as we are able to on this call into
* wherever we left off from the last time we were called.
*/
- ret = secure_raw_read(port, PqGSSRecvBuffer + PqGSSRecvLength,
- input.length - (PqGSSRecvLength - sizeof(uint32)));
+ ret = io_stream_next_read(self, PqGSSRecvBuffer + PqGSSRecvLength,
+ input.length - (PqGSSRecvLength - sizeof(uint32)), false);
/* If ret <= 0, secure_raw_read already set the correct errno */
if (ret <= 0)
return ret;
@@ -413,7 +426,8 @@ be_gssapi_read(Port *port, void *ptr, size_t len)
/*
* Read the specified number of bytes off the wire, waiting using
- * WaitLatchOrSocket if we would block.
+ * WaitLatchOrSocket if we would block. Only used during connecition setup
+ * before GSS is added to the io_stream.
*
* Results are read into PqGSSRecvBuffer.
*
@@ -430,7 +444,7 @@ read_or_wait(Port *port, ssize_t len)
*/
while (PqGSSRecvLength < len)
{
- ret = secure_raw_read(port, PqGSSRecvBuffer + PqGSSRecvLength, len - PqGSSRecvLength);
+ ret = io_stream_read(port->io_stream, PqGSSRecvBuffer + PqGSSRecvLength, len - PqGSSRecvLength, false);
/*
* If we got back an error and it wasn't just
@@ -465,7 +479,7 @@ read_or_wait(Port *port, ssize_t len)
*/
if (ret == 0)
{
- ret = secure_raw_read(port, PqGSSRecvBuffer + PqGSSRecvLength, len - PqGSSRecvLength);
+ ret = io_stream_read(port->io_stream, PqGSSRecvBuffer + PqGSSRecvLength, len - PqGSSRecvLength, false);
if (ret == 0)
return -1;
}
@@ -648,8 +662,10 @@ secure_open_gssapi(Port *port)
while (PqGSSSendNext < PqGSSSendLength)
{
- ret = secure_raw_write(port, PqGSSSendBuffer + PqGSSSendNext,
- PqGSSSendLength - PqGSSSendNext);
+ size_t count;
+
+ ret = io_stream_write(port->io_stream, PqGSSSendBuffer + PqGSSSendNext,
+ PqGSSSendLength - PqGSSSendNext, &count);
/*
* If we got back an error and it wasn't just
@@ -663,7 +679,7 @@ secure_open_gssapi(Port *port)
}
/* Wait and retry if we couldn't write yet */
- if (ret <= 0)
+ if (ret < 0 || count == 0)
{
WaitLatchOrSocket(MyLatch,
WL_SOCKET_WRITEABLE | WL_EXIT_ON_PM_DEATH,
@@ -671,7 +687,7 @@ secure_open_gssapi(Port *port)
continue;
}
- PqGSSSendNext += ret;
+ PqGSSSendNext += count;
}
/* Done sending the packet, reset our buffer */
@@ -701,6 +717,7 @@ secure_open_gssapi(Port *port)
pg_GSS_error(_("GSSAPI size check error"), major, minor);
return -1;
}
+ io_stream_add_layer(port->io_stream, &be_gssapi_processor, port);
port->gss->enc = true;
diff --git a/src/backend/libpq/be-secure-openssl.c b/src/backend/libpq/be-secure-openssl.c
index 22e3dc5a81..5c67fd46aa 100644
--- a/src/backend/libpq/be-secure-openssl.c
+++ b/src/backend/libpq/be-secure-openssl.c
@@ -59,7 +59,7 @@ openssl_tls_init_hook_typ openssl_tls_init_hook = default_openssl_tls_init;
static int my_sock_read(BIO *h, char *buf, int size);
static int my_sock_write(BIO *h, const char *buf, int size);
static BIO_METHOD *my_BIO_s_socket(void);
-static int my_SSL_set_fd(Port *port, int fd);
+static int my_SSL_set_fd(SSL *ssl, int fd, IoStreamLayer * layer);
static DH *load_dh_file(char *filename, bool isServerStart);
static DH *load_dh_buffer(const char *buffer, size_t len);
@@ -71,6 +71,16 @@ static bool initialize_dh(SSL_CTX *context, bool isServerStart);
static bool initialize_ecdh(SSL_CTX *context, bool isServerStart);
static const char *SSLerrmessage(unsigned long ecode);
+static ssize_t be_tls_read(IoStreamLayer * self, Port *port, void *ptr, size_t len, bool buffered_only);
+static int be_tls_write(IoStreamLayer * self, Port *port, void const *ptr, size_t len, size_t *bytes_written);
+static void be_tls_close(Port *port);
+
+IoStreamProcessor be_tls_processor = {
+ .read = (io_stream_read_func) be_tls_read,
+ .write = (io_stream_write_func) be_tls_write,
+ .destroy = (io_stream_destroy_func) be_tls_close
+};
+
static char *X509_NAME_to_cstring(X509_NAME *name);
static SSL_CTX *SSL_context = NULL;
@@ -440,7 +450,8 @@ be_tls_open_server(Port *port)
SSLerrmessage(ERR_get_error()))));
return -1;
}
- if (!my_SSL_set_fd(port, port->sock))
+ if (!my_SSL_set_fd(port->ssl, port->sock,
+ io_stream_add_layer(port->io_stream, &be_tls_processor, port)))
{
ereport(COMMERROR,
(errcode(ERRCODE_PROTOCOL_VIOLATION),
@@ -674,7 +685,7 @@ aloop:
return 0;
}
-void
+static void
be_tls_close(Port *port)
{
if (port->ssl)
@@ -704,14 +715,27 @@ be_tls_close(Port *port)
}
}
-ssize_t
-be_tls_read(Port *port, void *ptr, size_t len, int *waitfor)
+static ssize_t
+be_tls_read(IoStreamLayer * self, Port *port, void *ptr, size_t len, bool buffered_only)
{
ssize_t n;
int err;
unsigned long ecode;
+ port->waitfor = 0;
errno = 0;
+
+ if (buffered_only)
+ {
+ /*
+ * SSL_pending bytes are guaranteed to be available and readable
+ * without blocking
+ */
+ len = Min(len, SSL_pending(port->ssl));
+ if (len == 0)
+ return 0;
+ }
+
ERR_clear_error();
n = SSL_read(port->ssl, ptr, len);
err = SSL_get_error(port->ssl, n);
@@ -722,12 +746,12 @@ be_tls_read(Port *port, void *ptr, size_t len, int *waitfor)
/* a-ok */
break;
case SSL_ERROR_WANT_READ:
- *waitfor = WL_SOCKET_READABLE;
+ port->waitfor = WL_SOCKET_READABLE;
errno = EWOULDBLOCK;
n = -1;
break;
case SSL_ERROR_WANT_WRITE:
- *waitfor = WL_SOCKET_WRITEABLE;
+ port->waitfor = WL_SOCKET_WRITEABLE;
errno = EWOULDBLOCK;
n = -1;
break;
@@ -763,13 +787,14 @@ be_tls_read(Port *port, void *ptr, size_t len, int *waitfor)
return n;
}
-ssize_t
-be_tls_write(Port *port, void *ptr, size_t len, int *waitfor)
+static int
+be_tls_write(IoStreamLayer * self, Port *port, void const *ptr, size_t len, size_t *bytes_written)
{
ssize_t n;
int err;
unsigned long ecode;
+ port->waitfor = 0;
errno = 0;
ERR_clear_error();
n = SSL_write(port->ssl, ptr, len);
@@ -781,12 +806,12 @@ be_tls_write(Port *port, void *ptr, size_t len, int *waitfor)
/* a-ok */
break;
case SSL_ERROR_WANT_READ:
- *waitfor = WL_SOCKET_READABLE;
+ port->waitfor = WL_SOCKET_READABLE;
errno = EWOULDBLOCK;
n = -1;
break;
case SSL_ERROR_WANT_WRITE:
- *waitfor = WL_SOCKET_WRITEABLE;
+ port->waitfor = WL_SOCKET_WRITEABLE;
errno = EWOULDBLOCK;
n = -1;
break;
@@ -830,7 +855,16 @@ be_tls_write(Port *port, void *ptr, size_t len, int *waitfor)
break;
}
- return n;
+ if (n >= 0)
+ {
+ *bytes_written = n;
+ return 0;
+ }
+ else
+ {
+ *bytes_written = 0;
+ return n;
+ }
}
/* ------------------------------------------------------------ */
@@ -858,7 +892,7 @@ my_sock_read(BIO *h, char *buf, int size)
if (buf != NULL)
{
- res = secure_raw_read(((Port *) BIO_get_app_data(h)), buf, size);
+ res = io_stream_next_read(BIO_get_app_data(h), buf, size, false);
BIO_clear_retry_flags(h);
if (res <= 0)
{
@@ -876,11 +910,12 @@ my_sock_read(BIO *h, char *buf, int size)
static int
my_sock_write(BIO *h, const char *buf, int size)
{
- int res = 0;
+ int res;
+ size_t bytes_written;
- res = secure_raw_write(((Port *) BIO_get_app_data(h)), buf, size);
+ res = io_stream_next_write(BIO_get_app_data(h), buf, size, &bytes_written);
BIO_clear_retry_flags(h);
- if (res <= 0)
+ if (res < 0 || bytes_written == 0)
{
/* If we were interrupted, tell caller to retry */
if (errno == EINTR || errno == EWOULDBLOCK || errno == EAGAIN)
@@ -889,7 +924,7 @@ my_sock_write(BIO *h, const char *buf, int size)
}
}
- return res;
+ return res ? res : bytes_written;
}
static BIO_METHOD *
@@ -935,7 +970,7 @@ my_BIO_s_socket(void)
/* This should exactly match OpenSSL's SSL_set_fd except for using my BIO */
static int
-my_SSL_set_fd(Port *port, int fd)
+my_SSL_set_fd(SSL *ssl, int fd, IoStreamLayer * layer)
{
int ret = 0;
BIO *bio;
@@ -954,10 +989,10 @@ my_SSL_set_fd(Port *port, int fd)
SSLerr(SSL_F_SSL_SET_FD, ERR_R_BUF_LIB);
goto err;
}
- BIO_set_app_data(bio, port);
+ BIO_set_app_data(bio, layer);
BIO_set_fd(bio, fd, BIO_NOCLOSE);
- SSL_set_bio(port->ssl, bio, bio);
+ SSL_set_bio(ssl, bio, bio);
ret = 1;
err:
return ret;
diff --git a/src/backend/libpq/be-secure.c b/src/backend/libpq/be-secure.c
index a0f7084018..caee5b0556 100644
--- a/src/backend/libpq/be-secure.c
+++ b/src/backend/libpq/be-secure.c
@@ -125,219 +125,3 @@ secure_open_server(Port *port)
return r;
}
-
-/*
- * Close secure session.
- */
-void
-secure_close(Port *port)
-{
-#ifdef USE_SSL
- if (port->ssl_in_use)
- be_tls_close(port);
-#endif
-}
-
-/*
- * Read data from a secure connection.
- */
-ssize_t
-secure_read(Port *port, void *ptr, size_t len)
-{
- ssize_t n;
- int waitfor;
-
- /* Deal with any already-pending interrupt condition. */
- ProcessClientReadInterrupt(false);
-
-retry:
-#ifdef USE_SSL
- waitfor = 0;
- if (port->ssl_in_use)
- {
- n = be_tls_read(port, ptr, len, &waitfor);
- }
- else
-#endif
-#ifdef ENABLE_GSS
- if (port->gss && port->gss->enc)
- {
- n = be_gssapi_read(port, ptr, len);
- waitfor = WL_SOCKET_READABLE;
- }
- else
-#endif
- {
- n = secure_raw_read(port, ptr, len);
- waitfor = WL_SOCKET_READABLE;
- }
-
- /* In blocking mode, wait until the socket is ready */
- if (n < 0 && !port->noblock && (errno == EWOULDBLOCK || errno == EAGAIN))
- {
- WaitEvent event;
-
- Assert(waitfor);
-
- ModifyWaitEvent(FeBeWaitSet, FeBeWaitSetSocketPos, waitfor, NULL);
-
- WaitEventSetWait(FeBeWaitSet, -1 /* no timeout */ , &event, 1,
- WAIT_EVENT_CLIENT_READ);
-
- /*
- * If the postmaster has died, it's not safe to continue running,
- * because it is the postmaster's job to kill us if some other backend
- * exits uncleanly. Moreover, we won't run very well in this state;
- * helper processes like walwriter and the bgwriter will exit, so
- * performance may be poor. Finally, if we don't exit, pg_ctl will be
- * unable to restart the postmaster without manual intervention, so no
- * new connections can be accepted. Exiting clears the deck for a
- * postmaster restart.
- *
- * (Note that we only make this check when we would otherwise sleep on
- * our latch. We might still continue running for a while if the
- * postmaster is killed in mid-query, or even through multiple queries
- * if we never have to wait for read. We don't want to burn too many
- * cycles checking for this very rare condition, and this should cause
- * us to exit quickly in most cases.)
- */
- if (event.events & WL_POSTMASTER_DEATH)
- ereport(FATAL,
- (errcode(ERRCODE_ADMIN_SHUTDOWN),
- errmsg("terminating connection due to unexpected postmaster exit")));
-
- /* Handle interrupt. */
- if (event.events & WL_LATCH_SET)
- {
- ResetLatch(MyLatch);
- ProcessClientReadInterrupt(true);
-
- /*
- * We'll retry the read. Most likely it will return immediately
- * because there's still no data available, and we'll wait for the
- * socket to become ready again.
- */
- }
- goto retry;
- }
-
- /*
- * Process interrupts that happened during a successful (or non-blocking,
- * or hard-failed) read.
- */
- ProcessClientReadInterrupt(false);
-
- return n;
-}
-
-ssize_t
-secure_raw_read(Port *port, void *ptr, size_t len)
-{
- ssize_t n;
-
- /*
- * Try to read from the socket without blocking. If it succeeds we're
- * done, otherwise we'll wait for the socket using the latch mechanism.
- */
-#ifdef WIN32
- pgwin32_noblock = true;
-#endif
- n = recv(port->sock, ptr, len, 0);
-#ifdef WIN32
- pgwin32_noblock = false;
-#endif
-
- return n;
-}
-
-
-/*
- * Write data to a secure connection.
- */
-ssize_t
-secure_write(Port *port, void *ptr, size_t len)
-{
- ssize_t n;
- int waitfor;
-
- /* Deal with any already-pending interrupt condition. */
- ProcessClientWriteInterrupt(false);
-
-retry:
- waitfor = 0;
-#ifdef USE_SSL
- if (port->ssl_in_use)
- {
- n = be_tls_write(port, ptr, len, &waitfor);
- }
- else
-#endif
-#ifdef ENABLE_GSS
- if (port->gss && port->gss->enc)
- {
- n = be_gssapi_write(port, ptr, len);
- waitfor = WL_SOCKET_WRITEABLE;
- }
- else
-#endif
- {
- n = secure_raw_write(port, ptr, len);
- waitfor = WL_SOCKET_WRITEABLE;
- }
-
- if (n < 0 && !port->noblock && (errno == EWOULDBLOCK || errno == EAGAIN))
- {
- WaitEvent event;
-
- Assert(waitfor);
-
- ModifyWaitEvent(FeBeWaitSet, FeBeWaitSetSocketPos, waitfor, NULL);
-
- WaitEventSetWait(FeBeWaitSet, -1 /* no timeout */ , &event, 1,
- WAIT_EVENT_CLIENT_WRITE);
-
- /* See comments in secure_read. */
- if (event.events & WL_POSTMASTER_DEATH)
- ereport(FATAL,
- (errcode(ERRCODE_ADMIN_SHUTDOWN),
- errmsg("terminating connection due to unexpected postmaster exit")));
-
- /* Handle interrupt. */
- if (event.events & WL_LATCH_SET)
- {
- ResetLatch(MyLatch);
- ProcessClientWriteInterrupt(true);
-
- /*
- * We'll retry the write. Most likely it will return immediately
- * because there's still no buffer space available, and we'll wait
- * for the socket to become ready again.
- */
- }
- goto retry;
- }
-
- /*
- * Process interrupts that happened during a successful (or non-blocking,
- * or hard-failed) write.
- */
- ProcessClientWriteInterrupt(false);
-
- return n;
-}
-
-ssize_t
-secure_raw_write(Port *port, const void *ptr, size_t len)
-{
- ssize_t n;
-
-#ifdef WIN32
- pgwin32_noblock = true;
-#endif
- n = send(port->sock, ptr, len, 0);
-#ifdef WIN32
- pgwin32_noblock = false;
-#endif
-
- return n;
-}
diff --git a/src/backend/libpq/pqcomm.c b/src/backend/libpq/pqcomm.c
index 0084a9bf13..030686cc3b 100644
--- a/src/backend/libpq/pqcomm.c
+++ b/src/backend/libpq/pqcomm.c
@@ -31,6 +31,7 @@
* setup/teardown:
* StreamServerPort - Open postmaster's server port
* StreamConnection - Create new connection with client
+ * StreamSetupIo - Configures IO stream on connection
* StreamClose - Close a client/backend connection
* TouchSocketFiles - Protect socket files against /tmp cleaners
* pq_init - initialize libpq at backend startup
@@ -74,12 +75,15 @@
#endif
#include "common/ip.h"
+#include "common/io_stream.h"
#include "libpq/libpq.h"
#include "miscadmin.h"
#include "port/pg_bswap.h"
#include "storage/ipc.h"
+#include "tcop/tcopprot.h"
#include "utils/guc_hooks.h"
#include "utils/memutils.h"
+#include "utils/wait_event.h"
/*
* Cope with the various platform-specific ways to spell TCP keepalive socket
@@ -146,6 +150,15 @@ static int socket_putmessage(char msgtype, const char *s, size_t len);
static void socket_putmessage_noblock(char msgtype, const char *s, size_t len);
static int internal_putbytes(const char *s, size_t len);
static int internal_flush(void);
+static ssize_t socket_read(IoStreamLayer * self, Port *port, void *ptr, size_t len, bool buffered_only);
+static int socket_write(IoStreamLayer * self, Port *port, void const *ptr, size_t len, size_t *bytes_written);
+static ssize_t io_read_with_wait(Port *port, void *ptr, size_t len);
+static int io_write_with_wait(Port *port, char const *ptr, size_t len, size_t *bytes_written);
+
+IoStreamProcessor socket_processor = {
+ .read = (io_stream_read_func) socket_read,
+ .write = (io_stream_write_func) socket_write
+};
static int Lock_AF_UNIX(const char *unixSocketDir, const char *unixSocketPath);
static int Setup_AF_UNIX(const char *sock_path);
@@ -277,7 +290,8 @@ socket_close(int code, Datum arg)
* Cleanly shut down SSL layer. Nowhere else does a postmaster child
* call this, so this is safe when interrupting BackendInitialize().
*/
- secure_close(MyProcPort);
+ io_stream_destroy(MyProcPort->io_stream);
+ MyProcPort->io_stream = NULL;
/*
* Formerly we did an explicit close() here, but it seems better to
@@ -817,6 +831,30 @@ StreamConnection(pgsocket server_fd, Port *port)
return STATUS_OK;
}
+/* This must be called after the child process is launched or the data structures
+ * do not comre across correctly
+ */
+int
+StreamSetupIo(Port *port)
+{
+ if (port->io_stream)
+ {
+ ereport(LOG,
+ (errmsg("%s() failed: io_stream already configured", "io_stream_create")));
+ return STATUS_ERROR;
+ }
+ port->io_stream = io_stream_create();
+ if (!port->io_stream)
+ {
+ ereport(LOG,
+ (errmsg("%s() failed: %m", "io_stream_create")));
+ return STATUS_ERROR;
+ }
+ io_stream_add_layer(port->io_stream, &socket_processor, port);
+
+ return STATUS_OK;
+}
+
/*
* StreamClose -- close a client/backend connection
*
@@ -905,6 +943,193 @@ socket_set_nonblocking(bool nonblocking)
MyProcPort->noblock = nonblocking;
}
+static ssize_t
+socket_read(IoStreamLayer * self, Port *port, void *ptr, size_t len, bool buffered_only)
+{
+ ssize_t n;
+
+ if (buffered_only)
+ return 0;
+
+ /*
+ * Try to read from the socket without blocking. If it succeeds we're
+ * done, otherwise we'll wait for the socket using the latch mechanism.
+ */
+#ifdef WIN32
+ pgwin32_noblock = true;
+#endif
+ n = recv(port->sock, ptr, len, 0);
+#ifdef WIN32
+ pgwin32_noblock = false;
+#endif
+
+ return n;
+}
+
+static int
+socket_write(IoStreamLayer * self, Port *port, const void *ptr, size_t len, size_t *bytes_written)
+{
+ ssize_t n;
+
+#ifdef WIN32
+ pgwin32_noblock = true;
+#endif
+ n = send(port->sock, ptr, len, 0);
+#ifdef WIN32
+ pgwin32_noblock = false;
+#endif
+
+ if (n >= 0)
+ {
+ *bytes_written = n;
+ return 0;
+ }
+ else
+ {
+ *bytes_written = 0;
+ return n;
+ }
+}
+
+/*
+ * Read protocol-level data, processed through any intermediate streams like TLS
+ */
+static ssize_t
+io_read_with_wait(Port *port, void *ptr, size_t len)
+{
+ ssize_t n;
+
+ /* Deal with any already-pending interrupt condition. */
+ ProcessClientReadInterrupt(false);
+
+retry:
+ port->waitfor = WL_SOCKET_READABLE;
+ n = io_stream_read(port->io_stream, ptr, len, false);
+
+ /* In blocking mode, wait until the socket is ready */
+ if (n < 0 && !port->noblock && (errno == EWOULDBLOCK || errno == EAGAIN))
+ {
+ WaitEvent event;
+
+ Assert(port->waitfor);
+
+ ModifyWaitEvent(FeBeWaitSet, FeBeWaitSetSocketPos, port->waitfor, NULL);
+
+ WaitEventSetWait(FeBeWaitSet, -1 /* no timeout */ , &event, 1,
+ WAIT_EVENT_CLIENT_READ);
+
+ /*
+ * If the postmaster has died, it's not safe to continue running,
+ * because it is the postmaster's job to kill us if some other backend
+ * exits uncleanly. Moreover, we won't run very well in this state;
+ * helper processes like walwriter and the bgwriter will exit, so
+ * performance may be poor. Finally, if we don't exit, pg_ctl will be
+ * unable to restart the postmaster without manual intervention, so no
+ * new connections can be accepted. Exiting clears the deck for a
+ * postmaster restart.
+ *
+ * (Note that we only make this check when we would otherwise sleep on
+ * our latch. We might still continue running for a while if the
+ * postmaster is killed in mid-query, or even through multiple queries
+ * if we never have to wait for read. We don't want to burn too many
+ * cycles checking for this very rare condition, and this should cause
+ * us to exit quickly in most cases.)
+ */
+ if (event.events & WL_POSTMASTER_DEATH)
+ ereport(FATAL,
+ (errcode(ERRCODE_ADMIN_SHUTDOWN),
+ errmsg("terminating connection due to unexpected postmaster exit")));
+
+ /* Handle interrupt. */
+ if (event.events & WL_LATCH_SET)
+ {
+ ResetLatch(MyLatch);
+ ProcessClientReadInterrupt(true);
+
+ /*
+ * We'll retry the read. Most likely it will return immediately
+ * because there's still no data available, and we'll wait for the
+ * socket to become ready again.
+ */
+ }
+ goto retry;
+ }
+
+ /*
+ * Process interrupts that happened during a successful (or non-blocking,
+ * or hard-failed) read.
+ */
+ ProcessClientReadInterrupt(false);
+
+ return n;
+}
+
+
+/*
+ * Write protocol-level data to be processed through any intermediate streams like TLS
+ */
+static int
+io_write_with_wait(Port *port, char const *ptr, size_t len, size_t *bytes_processed)
+{
+ int rc;
+ size_t count = 0;
+
+ *bytes_processed = 0;
+
+ /* Deal with any already-pending interrupt condition. */
+ ProcessClientWriteInterrupt(false);
+
+retry:
+ port->waitfor = WL_SOCKET_WRITEABLE;
+
+ /*
+ * on retry it is possible some of the input will have already been
+ * processed, so make sure we offset our retries
+ */
+ rc = io_stream_write(port->io_stream, ptr + *bytes_processed, len - *bytes_processed, &count);
+ *bytes_processed += count;
+
+ if (rc < 0 && !port->noblock && (errno == EWOULDBLOCK || errno == EAGAIN))
+ {
+ WaitEvent event;
+
+ Assert(port->waitfor);
+
+ ModifyWaitEvent(FeBeWaitSet, FeBeWaitSetSocketPos, port->waitfor, NULL);
+
+ WaitEventSetWait(FeBeWaitSet, -1 /* no timeout */ , &event, 1,
+ WAIT_EVENT_CLIENT_WRITE);
+
+ /* See comments in secure_read. */
+ if (event.events & WL_POSTMASTER_DEATH)
+ ereport(FATAL,
+ (errcode(ERRCODE_ADMIN_SHUTDOWN),
+ errmsg("terminating connection due to unexpected postmaster exit")));
+
+ /* Handle interrupt. */
+ if (event.events & WL_LATCH_SET)
+ {
+ ResetLatch(MyLatch);
+ ProcessClientWriteInterrupt(true);
+
+ /*
+ * We'll retry the write. Most likely it will return immediately
+ * because there's still no buffer space available, and we'll wait
+ * for the socket to become ready again.
+ */
+ }
+ goto retry;
+ }
+
+ /*
+ * Process interrupts that happened during a successful (or non-blocking,
+ * or hard-failed) write.
+ */
+ ProcessClientWriteInterrupt(false);
+
+ return rc;
+}
+
/* --------------------------------
* pq_recvbuf - load some bytes into the input buffer
*
@@ -938,8 +1163,8 @@ pq_recvbuf(void)
errno = 0;
- r = secure_read(MyProcPort, PqRecvBuffer + PqRecvLength,
- PQ_RECV_BUFFER_SIZE - PqRecvLength);
+ r = io_read_with_wait(MyProcPort, PqRecvBuffer + PqRecvLength,
+ PQ_RECV_BUFFER_SIZE - PqRecvLength);
if (r < 0)
{
@@ -1035,7 +1260,7 @@ pq_getbyte_if_available(unsigned char *c)
errno = 0;
- r = secure_read(MyProcPort, c, 1);
+ r = io_read_with_wait(MyProcPort, c, 1);
if (r < 0)
{
/*
@@ -1351,11 +1576,15 @@ internal_flush(void)
while (bufptr < bufend)
{
- int r;
+ int rc;
+ size_t bytes_sent;
+ size_t available = bufend - bufptr;
- r = secure_write(MyProcPort, bufptr, bufend - bufptr);
+ rc = io_write_with_wait(MyProcPort, bufptr, available, &bytes_sent);
+ bufptr += bytes_sent;
+ PqSendStart += bytes_sent;
- if (r <= 0)
+ if (rc < 0 || (bytes_sent == 0 && available))
{
if (errno == EINTR)
continue; /* Ok if we were interrupted */
@@ -1400,8 +1629,6 @@ internal_flush(void)
}
last_reported_send_errno = 0; /* reset after any successful send */
- bufptr += r;
- PqSendStart += r;
}
PqSendStart = PqSendPointer = 0;
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index fb04e4dde3..edf28e36f1 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -4232,6 +4232,11 @@ BackendInitialize(Port *port)
/* Save port etc. for ps status */
MyProcPort = port;
+ if (StreamSetupIo(port))
+ ereport(ERROR,
+ (errcode(ERRCODE_INTERNAL_ERROR),
+ errmsg("Unable to configure backend I/O")));
+
/* Tell fd.c about the long-lived FD associated with the port */
ReserveExternalFD();
diff --git a/src/common/Makefile b/src/common/Makefile
index 2ba5069dca..a80f4b39a6 100644
--- a/src/common/Makefile
+++ b/src/common/Makefile
@@ -59,6 +59,7 @@ OBJS_COMMON = \
file_perm.o \
file_utils.o \
hashfn.o \
+ io_stream.o \
ip.o \
jsonapi.o \
keywords.o \
diff --git a/src/common/io_stream.c b/src/common/io_stream.c
new file mode 100644
index 0000000000..b15aca326d
--- /dev/null
+++ b/src/common/io_stream.c
@@ -0,0 +1,148 @@
+/*-------------------------------------------------------------------------
+ *
+ * io_stream.c
+ * functions related to managing layers of streaming IO.
+ * In general the base layer will work with raw sockets, and then
+ * additional layers will add features such as encryption and
+ * compression.
+ *
+ * Copyright (c) 2023, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/common/io_stream.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef FRONTEND
+#include "postgres.h"
+#else
+#include "postgres_fe.h"
+#endif
+
+#include <common/io_stream.h>
+
+#ifndef FRONTEND
+#include "utils/memutils.h"
+#define ALLOC(size) MemoryContextAlloc(TopMemoryContext, size)
+#define FREE(size) pfree(size)
+#else
+#define ALLOC(size) malloc(size)
+#define FREE(size) free(size)
+#endif
+
+struct IoStreamLayer
+{
+ IoStreamProcessor *processor;
+ void *context;
+ IoStreamLayer *next;
+};
+
+struct IoStream
+{
+ IoStreamLayer *layer;
+};
+
+IoStream *
+io_stream_create(void)
+{
+ IoStream *ret = ALLOC(sizeof(IoStream));
+
+ ret->layer = NULL;
+ return ret;
+}
+
+void
+io_stream_destroy(IoStream * arg)
+{
+ IoStreamLayer *layer;
+
+ if (arg == NULL)
+ return;
+
+ layer = arg->layer;
+ while (layer != NULL)
+ {
+ IoStreamLayer *next = layer->next;
+
+ if (layer->processor->destroy != NULL)
+ layer->processor->destroy(layer->context);
+ FREE(layer);
+ layer = next;
+ }
+ FREE(arg);
+}
+
+IoStreamLayer *
+io_stream_add_layer(IoStream * stream, IoStreamProcessor * processor, void *context)
+{
+ IoStreamLayer *layer = ALLOC(sizeof(IoStreamLayer));
+
+ layer->processor = processor;
+ layer->context = context;
+ layer->next = stream->layer;
+ stream->layer = layer;
+ return layer;
+}
+
+ssize_t
+io_stream_read(IoStream * stream, void *data, size_t size, bool buffered_only)
+{
+ if (stream->layer == NULL)
+ return -1;
+
+ return stream->layer->processor->read(stream->layer, stream->layer->context, data, size, buffered_only);
+}
+
+int
+io_stream_write(IoStream * stream, void const *data, size_t size, size_t *bytes_written)
+{
+ if (stream->layer == NULL)
+ return -1;
+
+ return stream->layer->processor->write(stream->layer, stream->layer->context, data, size, bytes_written);
+}
+
+bool
+io_stream_buffered_read_data(IoStream * stream)
+{
+ IoStreamLayer *layer;
+
+ for (layer = stream->layer; layer != NULL; layer = layer->next)
+ {
+ if (layer->processor->buffered_read_data != NULL && layer->processor->buffered_read_data(layer->context))
+ return true;
+ }
+ return false;
+}
+
+bool
+io_stream_buffered_write_data(IoStream * stream)
+{
+ IoStreamLayer *layer;
+
+ for (layer = stream->layer; layer != NULL; layer = layer->next)
+ {
+ if (layer->processor->buffered_write_data != NULL && layer->processor->buffered_write_data(layer->context))
+ return true;
+ }
+ return false;
+}
+
+ssize_t
+io_stream_next_read(IoStreamLayer * layer, void *data, size_t size, bool buffered_only)
+{
+ if (layer->next == NULL)
+ return -1;
+
+ return layer->next->processor->read(layer->next, layer->next->context, data, size, buffered_only);
+}
+
+int
+io_stream_next_write(IoStreamLayer * layer, void const *data, size_t size, size_t *bytes_written)
+{
+ if (layer->next == NULL)
+ return -1;
+
+ return layer->next->processor->write(layer->next, layer->next->context, data, size, bytes_written);
+}
diff --git a/src/common/meson.build b/src/common/meson.build
index 12fd43e87f..e25fa44133 100644
--- a/src/common/meson.build
+++ b/src/common/meson.build
@@ -13,6 +13,7 @@ common_sources = files(
'file_perm.c',
'file_utils.c',
'hashfn.c',
+ 'io_stream.c',
'ip.c',
'jsonapi.c',
'keywords.c',
diff --git a/src/include/common/io_stream.h b/src/include/common/io_stream.h
new file mode 100644
index 0000000000..9af88aa9ea
--- /dev/null
+++ b/src/include/common/io_stream.h
@@ -0,0 +1,131 @@
+/*-------------------------------------------------------------------------
+ *
+ * io_stream.c
+ * defintions for managing layers of streaming IO.
+ * In general the base layer will work with raw sockets, and then
+ * additional layers will add features such as encryption and
+ * compression.
+ *
+ * Copyright (c) 2023, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/include/common/io_stream.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef IO_STREAM_H
+#define IO_STREAM_H
+
+/* Opaque structs should only be interacted with through corresponding functions*/
+typedef struct IoStreamLayer IoStreamLayer;
+typedef struct IoStream IoStream;
+
+typedef ssize_t (*io_stream_read_func) (IoStreamLayer * self, void *context, void *data, size_t size, bool buffered_only);
+typedef int (*io_stream_write_func) (IoStreamLayer * self, void *context, void const *data, size_t size, size_t *bytes_written);
+typedef bool (*io_stream_predicate) (void *context);
+typedef void (*io_stream_destroy_func) (void *context);
+
+typedef struct IoStreamProcessor
+{
+ /*
+ * Required Should call io_stream_next_read with self either directly or
+ * indirectly to recieve bytes from the underlying layer of the stream
+ */
+ io_stream_read_func read;
+
+ /*
+ * Required Should call io_stream_next_write with self either directly or
+ * indirectly to write bytes to the underlying layer of the stream
+ */
+ io_stream_write_func write;
+
+ /*
+ * Optional Return true if this layer is holding buffered readable data
+ */
+ io_stream_predicate buffered_read_data;
+
+ /*
+ * Optional Return true if this layer is holding buffered writable data
+ */
+ io_stream_predicate buffered_write_data;
+
+ /*
+ * Optional will be called as part of io_stream_destroy when cleaning up
+ * the stream
+ */
+ io_stream_destroy_func destroy;
+} IoStreamProcessor;
+
+/*
+ * Allocate a new IoStream and return the address to the caller. IoStreams should always be destroyed with
+ * the corresponding io_stream_destroy function
+ */
+extern IoStream * io_stream_create(void);
+/*
+ * Adds new processors to the IO stream
+ *
+ * processorDescription contains collection of function pointers for this layer
+ * context (optional) pointer with context to be used in reader/writer
+ *
+ * Returns the newly created processor
+ * */
+extern IoStreamLayer * io_stream_add_layer(IoStream * stream, IoStreamProcessor * processorDescription, void *context);
+/*
+ * Destroy an IoStream, freeing all associated memory
+ */
+extern void io_stream_destroy(IoStream * stream);
+
+/*
+ * Read data from the stream
+ *
+ * Reads at most size bytes into the buffer pointed to by data, and returns
+ * the number of bytes read. If buffered_only is true, then only data that
+ * was stored in an in-process buffer will be returned and this function will
+ * not block. In that case, a return value of 0 simply means there was no
+ * buffered data available and does not mean the stream has reached EOF.
+ */
+extern ssize_t io_stream_read(IoStream * stream, void *data, size_t size, bool buffered_only);
+
+/*
+ * Write data to the stream
+ *
+ * Writes at most size bytes from the buffer pointed to by data, and returns
+ * true on success, false on failure, with the specific error in errno. The
+ * count of bytes written from data will be stored in bytes_written and may
+ * be non-zero even on failure
+ */
+extern int io_stream_write(IoStream * stream, void const *data, size_t size, size_t *bytes_written);
+
+/*
+ * Check if there is data buffered in memory waiting to be read (e.g. if
+ * compression is in use and more uncompressed data was read than fit into
+ * the provided buffer)
+ */
+extern bool io_stream_buffered_read_data(IoStream * stream);
+
+/*
+ * Check if there is data buffered in memory waiting to be written to the underlying backing store
+ */
+extern bool io_stream_buffered_write_data(IoStream * stream);
+
+/*
+ * Read data from the next layer of the stream
+ * (to be used by io_stream_read_func)
+ *
+ * Reads at most size bytes into the buffer pointed to by data, and returns
+ * the number of bytes read
+ */
+extern ssize_t io_stream_next_read(IoStreamLayer * layer, void *data, size_t size, bool buffered_only);
+
+/*
+ * Write data to the next layer of the stream
+ * (to be used by io_stream_write_func)
+ *
+ * Writes at most size bytes from the buffer pointed to by data, and returns
+ * true on success, false on failure, with the specific error in errno. The
+ * count of bytes written from data will be stored in bytes_written and may
+ * be non-zero even on failure
+ */
+extern int io_stream_next_write(IoStreamLayer * layer, void const *data, size_t size, size_t *bytes_written);
+
+#endif /* //IO_STREAM_H */
diff --git a/src/include/libpq/libpq-be.h b/src/include/libpq/libpq-be.h
index c57ed12fb6..87ba7f5ea0 100644
--- a/src/include/libpq/libpq-be.h
+++ b/src/include/libpq/libpq-be.h
@@ -53,6 +53,7 @@ typedef struct
#endif
#endif /* ENABLE_SSPI */
+#include "common/io_stream.h"
#include "datatype/timestamp.h"
#include "libpq/hba.h"
#include "libpq/pqcomm.h"
@@ -145,8 +146,11 @@ typedef struct ClientConnectionInfo
typedef struct Port
{
+ IoStream *io_stream;
pgsocket sock; /* File descriptor */
bool noblock; /* is the socket in non-blocking mode? */
+ int waitfor; /* Events to wait on the socket for after
+ * attempted read/write */
ProtocolVersion proto; /* FE/BE protocol version */
SockAddr laddr; /* local addr (postmaster) */
SockAddr raddr; /* remote addr (client) */
@@ -274,21 +278,6 @@ extern void be_tls_destroy(void);
*/
extern int be_tls_open_server(Port *port);
-/*
- * Close SSL connection.
- */
-extern void be_tls_close(Port *port);
-
-/*
- * Read data from a secure connection.
- */
-extern ssize_t be_tls_read(Port *port, void *ptr, size_t len, int *waitfor);
-
-/*
- * Write data to a secure connection.
- */
-extern ssize_t be_tls_write(Port *port, void *ptr, size_t len, int *waitfor);
-
/*
* Return information about the SSL connection.
*/
@@ -324,10 +313,6 @@ extern bool be_gssapi_get_auth(Port *port);
extern bool be_gssapi_get_enc(Port *port);
extern const char *be_gssapi_get_princ(Port *port);
extern bool be_gssapi_get_delegation(Port *port);
-
-/* Read and write to a GSSAPI-encrypted connection. */
-extern ssize_t be_gssapi_read(Port *port, void *ptr, size_t len);
-extern ssize_t be_gssapi_write(Port *port, void *ptr, size_t len);
#endif /* ENABLE_GSS */
extern PGDLLIMPORT ProtocolVersion FrontendProtocol;
diff --git a/src/include/libpq/libpq.h b/src/include/libpq/libpq.h
index a6104d8cd0..3612280146 100644
--- a/src/include/libpq/libpq.h
+++ b/src/include/libpq/libpq.h
@@ -68,6 +68,7 @@ extern int StreamServerPort(int family, const char *hostName,
unsigned short portNumber, const char *unixSocketDir,
pgsocket ListenSocket[], int *NumListenSockets, int MaxListen);
extern int StreamConnection(pgsocket server_fd, Port *port);
+extern int StreamSetupIo(Port *port);
extern void StreamClose(pgsocket sock);
extern void TouchSocketFiles(void);
extern void RemoveSocketFiles(void);
@@ -104,11 +105,6 @@ extern int secure_initialize(bool isServerStart);
extern bool secure_loaded_verify_locations(void);
extern void secure_destroy(void);
extern int secure_open_server(Port *port);
-extern void secure_close(Port *port);
-extern ssize_t secure_read(Port *port, void *ptr, size_t len);
-extern ssize_t secure_write(Port *port, void *ptr, size_t len);
-extern ssize_t secure_raw_read(Port *port, void *ptr, size_t len);
-extern ssize_t secure_raw_write(Port *port, const void *ptr, size_t len);
/*
* prototypes for functions in be-secure-gssapi.c
diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c
index bf83a9b569..a0f12e62af 100644
--- a/src/interfaces/libpq/fe-connect.c
+++ b/src/interfaces/libpq/fe-connect.c
@@ -136,6 +136,55 @@ static int ldapServiceLookup(const char *purl, PQconninfoOption *options,
#define DefaultGSSMode "disable"
#endif
+/*
+ * Macros to handle disabling and then restoring the state of SIGPIPE handling.
+ * On Windows, these are all no-ops since there's no SIGPIPEs.
+ */
+
+#ifndef WIN32
+
+#define SIGPIPE_MASKED(conn) ((conn)->sigpipe_so || (conn)->sigpipe_flag)
+
+struct sigpipe_info
+{
+ sigset_t oldsigmask;
+ bool sigpipe_pending;
+ bool got_epipe;
+};
+
+#define DECLARE_SIGPIPE_INFO(spinfo) struct sigpipe_info spinfo
+
+#define DISABLE_SIGPIPE(conn, spinfo, failaction) \
+do { \
+(spinfo).got_epipe = false; \
+if (!SIGPIPE_MASKED(conn)) \
+{ \
+if (pq_block_sigpipe(&(spinfo).oldsigmask, \
+&(spinfo).sigpipe_pending) < 0) \
+failaction; \
+} \
+} while (0)
+
+#define REMEMBER_EPIPE(spinfo, cond) \
+do { \
+if (cond) \
+(spinfo).got_epipe = true; \
+} while (0)
+
+#define RESTORE_SIGPIPE(conn, spinfo) \
+do { \
+if (!SIGPIPE_MASKED(conn)) \
+pq_reset_sigpipe(&(spinfo).oldsigmask, (spinfo).sigpipe_pending, \
+(spinfo).got_epipe); \
+} while (0)
+#else /* WIN32 */
+
+#define DECLARE_SIGPIPE_INFO(spinfo)
+#define DISABLE_SIGPIPE(conn, spinfo, failaction)
+#define REMEMBER_EPIPE(spinfo, cond)
+#define RESTORE_SIGPIPE(conn, spinfo)
+#endif /* WIN32 */
+
/* ----------
* Definition of the conninfo parameters and their fallback resources.
*
@@ -445,6 +494,12 @@ static bool sslVerifyProtocolVersion(const char *version);
static bool sslVerifyProtocolRange(const char *min, const char *max);
static bool parse_int_param(const char *value, int *result, PGconn *conn,
const char *context);
+static ssize_t socket_read(IoStreamLayer * self, PGconn *conn, void *ptr, size_t len, bool buffered_only);
+static int socket_write(IoStreamLayer * self, PGconn *conn, void const *ptr, size_t len, size_t *bytes_written);
+IoStreamProcessor socket_processor = {
+ .read = (io_stream_read_func) socket_read,
+ .write = (io_stream_write_func) socket_write
+};
/* global variable because fe-auth.c needs to access it */
@@ -466,9 +521,8 @@ pgthreadlock_t pg_g_threadlock = default_threadlock;
void
pqDropConnection(PGconn *conn, bool flushInput)
{
- /* Drop any SSL state */
- pqsecure_close(conn);
-
+ io_stream_destroy(conn->io_stream);
+ conn->io_stream = NULL;
/* Close the socket itself */
if (conn->sock != PGINVALID_SOCKET)
closesocket(conn->sock);
@@ -2879,6 +2933,9 @@ keep_going: /* We will come back to here until there is
sock_type |= SOCK_NONBLOCK;
#endif
conn->sock = socket(addr_cur->family, sock_type, 0);
+ conn->io_stream = io_stream_create();
+ io_stream_add_layer(conn->io_stream, &socket_processor, conn);
+
if (conn->sock == PGINVALID_SOCKET)
{
int errorno = SOCK_ERRNO;
@@ -7829,3 +7886,283 @@ PQregisterThreadLock(pgthreadlock_t newhandler)
return prev;
}
+
+static ssize_t
+socket_read(IoStreamLayer * self, PGconn *conn, void *ptr, size_t len, bool buffered_only)
+{
+ ssize_t n;
+ int result_errno = 0;
+ char sebuf[PG_STRERROR_R_BUFLEN];
+
+ SOCK_ERRNO_SET(0);
+
+ if (buffered_only)
+ return 0;
+
+ n = recv(conn->sock, ptr, len, 0);
+
+ if (n < 0)
+ {
+ result_errno = SOCK_ERRNO;
+
+ /* Set error message if appropriate */
+ switch (result_errno)
+ {
+#ifdef EAGAIN
+ case EAGAIN:
+#endif
+#if defined(EWOULDBLOCK) && (!defined(EAGAIN) || (EWOULDBLOCK != EAGAIN))
+ case EWOULDBLOCK:
+#endif
+ case EINTR:
+ /* no error message, caller is expected to retry */
+ break;
+
+ case EPIPE:
+ case ECONNRESET:
+ libpq_append_conn_error(conn, "server closed the connection unexpectedly\n"
+ "\tThis probably means the server terminated abnormally\n"
+ "\tbefore or while processing the request.");
+ break;
+
+ case 0:
+ /* If errno didn't get set, treat it as regular EOF */
+ n = 0;
+ break;
+
+ default:
+ libpq_append_conn_error(conn, "could not receive data from server: %s",
+ SOCK_STRERROR(result_errno,
+ sebuf, sizeof(sebuf)));
+ break;
+ }
+ }
+
+ /* ensure we return the intended errno to caller */
+ SOCK_ERRNO_SET(result_errno);
+
+ return n;
+}
+
+/*
+ * Socket-level implementation of data writing.
+ *
+ * This is used directly for an unencrypted connection. For encrypted
+ * connections, this is wrapped by higher layers through IO stream
+ *
+ * This function reports failure (i.e., returns a negative result) only
+ * for retryable errors such as EINTR. Looping for such cases is to be
+ * handled at some outer level, maybe all the way up to the application.
+ * For hard failures, we set conn->write_failed and store an error message
+ * in conn->write_err_msg, but then claim to have written the data anyway.
+ * This is because we don't want to report write failures so long as there
+ * is a possibility of reading from the server and getting an error message
+ * that could explain why the connection dropped. Many TCP stacks have
+ * race conditions such that a write failure may or may not be reported
+ * before all incoming data has been read.
+ *
+ * Note that this error behavior happens below the SSL management level when
+ * we are using SSL. That's because at least some versions of OpenSSL are
+ * too quick to report a write failure when there's still a possibility to
+ * get a more useful error from the server.
+ */
+static int
+socket_write(IoStreamLayer * self, PGconn *conn, void const *ptr, size_t len, size_t *bytes_written)
+{
+ ssize_t n;
+ int flags = 0;
+ int result_errno = 0;
+ char msgbuf[1024];
+ char sebuf[PG_STRERROR_R_BUFLEN];
+
+ DECLARE_SIGPIPE_INFO(spinfo);
+
+ /*
+ * If we already had a write failure, we will never again try to send data
+ * on that connection. Even if the kernel would let us, we've probably
+ * lost message boundary sync with the server. conn->write_failed
+ * therefore persists until the connection is reset, and we just discard
+ * all data presented to be written.
+ */
+ if (conn->write_failed)
+ return len;
+
+#ifdef MSG_NOSIGNAL
+ if (conn->sigpipe_flag)
+ flags |= MSG_NOSIGNAL;
+
+retry_masked:
+#endif /* MSG_NOSIGNAL */
+
+ DISABLE_SIGPIPE(conn, spinfo, return -1);
+
+ n = send(conn->sock, ptr, len, flags);
+
+ if (n < 0)
+ {
+ *bytes_written = 0;
+ result_errno = SOCK_ERRNO;
+
+ /*
+ * If we see an EINVAL, it may be because MSG_NOSIGNAL isn't available
+ * on this machine. So, clear sigpipe_flag so we don't try the flag
+ * again, and retry the send().
+ */
+#ifdef MSG_NOSIGNAL
+ if (flags != 0 && result_errno == EINVAL)
+ {
+ conn->sigpipe_flag = false;
+ flags = 0;
+ goto retry_masked;
+ }
+#endif /* MSG_NOSIGNAL */
+
+ /* Set error message if appropriate */
+ switch (result_errno)
+ {
+#ifdef EAGAIN
+ case EAGAIN:
+#endif
+#if defined(EWOULDBLOCK) && (!defined(EAGAIN) || (EWOULDBLOCK != EAGAIN))
+ case EWOULDBLOCK:
+#endif
+ case EINTR:
+ /* no error message, caller is expected to retry */
+ break;
+
+ case EPIPE:
+ /* Set flag for EPIPE */
+ REMEMBER_EPIPE(spinfo, true);
+
+ /* FALL THRU */
+
+ case ECONNRESET:
+ conn->write_failed = true;
+ /* Store error message in conn->write_err_msg, if possible */
+ /* (strdup failure is OK, we'll cope later) */
+ snprintf(msgbuf, sizeof(msgbuf),
+ libpq_gettext("server closed the connection unexpectedly\n"
+ "\tThis probably means the server terminated abnormally\n"
+ "\tbefore or while processing the request."));
+ /* keep newline out of translated string */
+ strlcat(msgbuf, "\n", sizeof(msgbuf));
+ conn->write_err_msg = strdup(msgbuf);
+ /* Now claim the write succeeded */
+ n = len;
+ break;
+
+ default:
+ conn->write_failed = true;
+ /* Store error message in conn->write_err_msg, if possible */
+ /* (strdup failure is OK, we'll cope later) */
+ snprintf(msgbuf, sizeof(msgbuf),
+ libpq_gettext("could not send data to server: %s"),
+ SOCK_STRERROR(result_errno,
+ sebuf, sizeof(sebuf)));
+ /* keep newline out of translated string */
+ strlcat(msgbuf, "\n", sizeof(msgbuf));
+ conn->write_err_msg = strdup(msgbuf);
+ /* Now claim the write succeeded */
+ n = len;
+ break;
+ }
+ }
+ else
+ {
+ *bytes_written = n;
+ n = 0;
+ }
+
+ RESTORE_SIGPIPE(conn, spinfo);
+
+ /* ensure we return the intended errno to caller */
+ SOCK_ERRNO_SET(result_errno);
+
+ return n;
+}
+
+#if !defined(WIN32)
+
+/*
+ * Block SIGPIPE for this thread. This prevents send()/write() from exiting
+ * the application.
+ */
+int
+pq_block_sigpipe(sigset_t *osigset, bool *sigpipe_pending)
+{
+ sigset_t sigpipe_sigset;
+ sigset_t sigset;
+
+ sigemptyset(&sigpipe_sigset);
+ sigaddset(&sigpipe_sigset, SIGPIPE);
+
+ /* Block SIGPIPE and save previous mask for later reset */
+ SOCK_ERRNO_SET(pthread_sigmask(SIG_BLOCK, &sigpipe_sigset, osigset));
+ if (SOCK_ERRNO)
+ return -1;
+
+ /* We can have a pending SIGPIPE only if it was blocked before */
+ if (sigismember(osigset, SIGPIPE))
+ {
+ /* Is there a pending SIGPIPE? */
+ if (sigpending(&sigset) != 0)
+ return -1;
+
+ if (sigismember(&sigset, SIGPIPE))
+ *sigpipe_pending = true;
+ else
+ *sigpipe_pending = false;
+ }
+ else
+ *sigpipe_pending = false;
+
+ return 0;
+}
+
+/*
+ * Discard any pending SIGPIPE and reset the signal mask.
+ *
+ * Note: we are effectively assuming here that the C library doesn't queue
+ * up multiple SIGPIPE events. If it did, then we'd accidentally leave
+ * ours in the queue when an event was already pending and we got another.
+ * As long as it doesn't queue multiple events, we're OK because the caller
+ * can't tell the difference.
+ *
+ * The caller should say got_epipe = false if it is certain that it
+ * didn't get an EPIPE error; in that case we'll skip the clear operation
+ * and things are definitely OK, queuing or no. If it got one or might have
+ * gotten one, pass got_epipe = true.
+ *
+ * We do not want this to change errno, since if it did that could lose
+ * the error code from a preceding send(). We essentially assume that if
+ * we were able to do pq_block_sigpipe(), this can't fail.
+ */
+void
+pq_reset_sigpipe(sigset_t *osigset, bool sigpipe_pending, bool got_epipe)
+{
+ int save_errno = SOCK_ERRNO;
+ int signo;
+ sigset_t sigset;
+
+ /* Clear SIGPIPE only if none was pending */
+ if (got_epipe && !sigpipe_pending)
+ {
+ if (sigpending(&sigset) == 0 &&
+ sigismember(&sigset, SIGPIPE))
+ {
+ sigset_t sigpipe_sigset;
+
+ sigemptyset(&sigpipe_sigset);
+ sigaddset(&sigpipe_sigset, SIGPIPE);
+
+ sigwait(&sigpipe_sigset, &signo);
+ }
+ }
+
+ /* Restore saved block mask */
+ pthread_sigmask(SIG_SETMASK, osigset, NULL);
+
+ SOCK_ERRNO_SET(save_errno);
+}
+
+#endif /* !WIN32 */
diff --git a/src/interfaces/libpq/fe-misc.c b/src/interfaces/libpq/fe-misc.c
index 660cdec93c..6772f2876d 100644
--- a/src/interfaces/libpq/fe-misc.c
+++ b/src/interfaces/libpq/fe-misc.c
@@ -614,8 +614,8 @@ pqReadData(PGconn *conn)
/* OK, try to read some data */
retry3:
- nread = pqsecure_read(conn, conn->inBuffer + conn->inEnd,
- conn->inBufSize - conn->inEnd);
+ nread = io_stream_read(conn->io_stream, conn->inBuffer + conn->inEnd,
+ conn->inBufSize - conn->inEnd, false);
if (nread < 0)
{
switch (SOCK_ERRNO)
@@ -709,8 +709,8 @@ retry3:
* arrived.
*/
retry4:
- nread = pqsecure_read(conn, conn->inBuffer + conn->inEnd,
- conn->inBufSize - conn->inEnd);
+ nread = io_stream_read(conn->io_stream, conn->inBuffer + conn->inEnd,
+ conn->inBufSize - conn->inEnd, false);
if (nread < 0)
{
switch (SOCK_ERRNO)
@@ -824,10 +824,11 @@ pqSendSome(PGconn *conn, int len)
/* while there's still data to send */
while (len > 0)
{
- int sent;
+ size_t sent;
+ int rc;
#ifndef WIN32
- sent = pqsecure_write(conn, ptr, len);
+ rc = io_stream_write(conn->io_stream, ptr, len, &sent);
#else
/*
@@ -835,10 +836,13 @@ pqSendSome(PGconn *conn, int len)
* failure-point appears to be different in different versions of
* Windows, but 64k should always be safe.
*/
- sent = pqsecure_write(conn, ptr, Min(len, 65536));
+ rc = io_stream_write(conn->io_stream, ptr, Min(len, 65536), &sent);
#endif
+ ptr += sent;
+ len -= sent;
+ remaining -= sent;
- if (sent < 0)
+ if (rc < 0)
{
/* Anything except EAGAIN/EWOULDBLOCK/EINTR is trouble */
switch (SOCK_ERRNO)
@@ -878,12 +882,6 @@ pqSendSome(PGconn *conn, int len)
return -1;
}
}
- else
- {
- ptr += sent;
- len -= sent;
- remaining -= sent;
- }
if (len > 0)
{
@@ -1048,14 +1046,11 @@ pqSocketCheck(PGconn *conn, int forRead, int forWrite, time_t end_time)
return -1;
}
-#ifdef USE_SSL
- /* Check for SSL library buffering read bytes */
- if (forRead && conn->ssl_in_use && pgtls_read_pending(conn))
+ if (forRead && io_stream_buffered_read_data(conn->io_stream))
{
/* short-circuit the select */
return 1;
}
-#endif
/* We will retry as long as we get EINTR */
do
diff --git a/src/interfaces/libpq/fe-secure-gssapi.c b/src/interfaces/libpq/fe-secure-gssapi.c
index ea8b0020d2..85cec6ced3 100644
--- a/src/interfaces/libpq/fe-secure-gssapi.c
+++ b/src/interfaces/libpq/fe-secure-gssapi.c
@@ -69,6 +69,13 @@
#define PqGSSResultNext (conn->gss_ResultNext)
#define PqGSSMaxPktSize (conn->gss_MaxPktSize)
+static ssize_t pg_GSS_read(IoStreamLayer * self, PGconn *conn, void *ptr, size_t len, bool buffered_only);
+static int pg_GSS_write(IoStreamLayer * self, PGconn *conn, const void *ptr, size_t len, size_t *bytes_written);
+
+IoStreamProcessor pg_GSS_processor = {
+ .read = (io_stream_read_func) pg_GSS_read,
+ .write = (io_stream_write_func) pg_GSS_write
+};
/*
* Attempt to write len bytes of data from ptr to a GSSAPI-encrypted connection.
@@ -82,8 +89,8 @@
* For retryable errors, caller should call again (passing the same or more
* data) once the socket is ready.
*/
-ssize_t
-pg_GSS_write(PGconn *conn, const void *ptr, size_t len)
+static int
+pg_GSS_write(IoStreamLayer * self, PGconn *conn, const void *ptr, size_t len, size_t *bytes_written)
{
OM_uint32 major,
minor;
@@ -94,6 +101,8 @@ pg_GSS_write(PGconn *conn, const void *ptr, size_t len)
size_t bytes_encrypted;
gss_ctx_id_t gctx = conn->gctx;
+ *bytes_written = 0;
+
/*
* When we get a retryable failure, we must not tell the caller we have
* successfully transmitted everything, else it won't retry. For
@@ -124,7 +133,7 @@ pg_GSS_write(PGconn *conn, const void *ptr, size_t len)
/*
* Loop through encrypting data and sending it out until it's all done or
- * pqsecure_raw_write() complains (which would likely mean that the socket
+ * io_stream_next_write() complains (which would likely mean that the socket
* is non-blocking and the requested send() would block, or there was some
* kind of actual error).
*/
@@ -141,20 +150,21 @@ pg_GSS_write(PGconn *conn, const void *ptr, size_t len)
*/
if (PqGSSSendLength)
{
- ssize_t retval;
+ int retval;
+ size_t count;
ssize_t amount = PqGSSSendLength - PqGSSSendNext;
- retval = pqsecure_raw_write(conn, PqGSSSendBuffer + PqGSSSendNext, amount);
- if (retval <= 0)
+ retval = io_stream_next_write(self, PqGSSSendBuffer + PqGSSSendNext, amount, &count);
+ if (retval < 0 || count == 0)
return retval;
/*
* Check if this was a partial write, and if so, move forward that
* far in our buffer and try again.
*/
- if (retval < amount)
+ if (count < amount)
{
- PqGSSSendNext += retval;
+ PqGSSSendNext += count;
continue;
}
@@ -235,7 +245,8 @@ pg_GSS_write(PGconn *conn, const void *ptr, size_t len)
/* We're reporting all the data as sent, so reset PqGSSSendConsumed. */
PqGSSSendConsumed = 0;
- ret = bytes_encrypted;
+ ret = 0;
+ *bytes_written = bytes_encrypted;
cleanup:
/* Release GSSAPI buffer storage, if we didn't already */
@@ -255,8 +266,8 @@ cleanup:
* error, a message is added to conn->errorMessage. For retryable errors,
* caller should call again once the socket is ready.
*/
-ssize_t
-pg_GSS_read(PGconn *conn, void *ptr, size_t len)
+static ssize_t
+pg_GSS_read(IoStreamLayer * self, PGconn *conn, void *ptr, size_t len, bool buffered_only)
{
OM_uint32 major,
minor;
@@ -266,6 +277,9 @@ pg_GSS_read(PGconn *conn, void *ptr, size_t len)
size_t bytes_returned = 0;
gss_ctx_id_t gctx = conn->gctx;
+ if (buffered_only)
+ return 0;
+
/*
* The plan here is to read one incoming encrypted packet into
* PqGSSRecvBuffer, decrypt it into PqGSSResultBuffer, and then dole out
@@ -322,10 +336,10 @@ pg_GSS_read(PGconn *conn, void *ptr, size_t len)
/* Collect the length if we haven't already */
if (PqGSSRecvLength < sizeof(uint32))
{
- ret = pqsecure_raw_read(conn, PqGSSRecvBuffer + PqGSSRecvLength,
- sizeof(uint32) - PqGSSRecvLength);
+ ret = io_stream_next_read(self, PqGSSRecvBuffer + PqGSSRecvLength,
+ sizeof(uint32) - PqGSSRecvLength, false);
- /* If ret <= 0, pqsecure_raw_read already set the correct errno */
+ /* If ret <= 0, io_stream_next_read already set the correct errno */
if (ret <= 0)
return ret;
@@ -355,9 +369,9 @@ pg_GSS_read(PGconn *conn, void *ptr, size_t len)
* Read as much of the packet as we are able to on this call into
* wherever we left off from the last time we were called.
*/
- ret = pqsecure_raw_read(conn, PqGSSRecvBuffer + PqGSSRecvLength,
- input.length - (PqGSSRecvLength - sizeof(uint32)));
- /* If ret <= 0, pqsecure_raw_read already set the correct errno */
+ ret = io_stream_next_read(self, PqGSSRecvBuffer + PqGSSRecvLength,
+ input.length - (PqGSSRecvLength - sizeof(uint32)), false);
+ /* If ret <= 0, io_stream_next_read already set the correct errno */
if (ret <= 0)
return ret;
@@ -418,16 +432,17 @@ cleanup:
}
/*
- * Simple wrapper for reading from pqsecure_raw_read.
+ * Simple wrapper for reading from io_stream_read. Only used during connecition setup
+ * before GSS is added to the io_stream.
*
- * This takes the same arguments as pqsecure_raw_read, plus an output parameter
+ * This takes the same arguments as io_stream_read, plus an output parameter
* to return the number of bytes read. This handles if blocking would occur and
* if we detect EOF on the connection.
*/
static PostgresPollingStatusType
gss_read(PGconn *conn, void *recv_buffer, size_t length, ssize_t *ret)
{
- *ret = pqsecure_raw_read(conn, recv_buffer, length);
+ *ret = io_stream_read(conn->io_stream, recv_buffer, length, false);
if (*ret < 0)
{
if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)
@@ -447,7 +462,7 @@ gss_read(PGconn *conn, void *recv_buffer, size_t length, ssize_t *ret)
if (!result)
return PGRES_POLLING_READING;
- *ret = pqsecure_raw_read(conn, recv_buffer, length);
+ *ret = io_stream_read(conn->io_stream, recv_buffer, length, false);
if (*ret < 0)
{
if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)
@@ -506,8 +521,9 @@ pqsecure_open_gss(PGconn *conn)
if (PqGSSSendLength)
{
ssize_t amount = PqGSSSendLength - PqGSSSendNext;
+ size_t count;
- ret = pqsecure_raw_write(conn, PqGSSSendBuffer + PqGSSSendNext, amount);
+ ret = io_stream_write(conn->io_stream, PqGSSSendBuffer + PqGSSSendNext, amount, &count);
if (ret < 0)
{
if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)
@@ -516,7 +532,7 @@ pqsecure_open_gss(PGconn *conn)
return PGRES_POLLING_FAILED;
}
- if (ret < amount)
+ if (count < amount)
{
PqGSSSendNext += ret;
return PGRES_POLLING_WRITING;
@@ -662,6 +678,8 @@ pqsecure_open_gss(PGconn *conn)
*/
conn->gssenc = true;
conn->gssapi_used = true;
+ io_stream_add_layer(conn->io_stream, &pg_GSS_processor, conn);
+
/* Clean up */
gss_release_cred(&minor, &conn->gcred);
diff --git a/src/interfaces/libpq/fe-secure-openssl.c b/src/interfaces/libpq/fe-secure-openssl.c
index 2b221e7d15..b9706cb151 100644
--- a/src/interfaces/libpq/fe-secure-openssl.c
+++ b/src/interfaces/libpq/fe-secure-openssl.c
@@ -81,8 +81,17 @@ static int PQssl_passwd_cb(char *buf, int size, int rwflag, void *userdata);
static int my_sock_read(BIO *h, char *buf, int size);
static int my_sock_write(BIO *h, const char *buf, int size);
static BIO_METHOD *my_BIO_s_socket(void);
-static int my_SSL_set_fd(PGconn *conn, int fd);
-
+static int my_SSL_set_fd(SSL *ssl, int fd, IoStreamLayer * layer);
+static ssize_t pgtls_read(IoStreamLayer * self, PGconn *conn, void *ptr, size_t len, bool buffered_only);
+static int pgtls_write(IoStreamLayer * self, PGconn *conn, void const *ptr, size_t len, size_t *bytes_written);
+static bool pgtls_read_pending(PGconn *conn);
+static void pgtls_close(PGconn *conn);
+IoStreamProcessor pgtls_processor = {
+ .read = (io_stream_read_func) pgtls_read,
+ .write = (io_stream_write_func) pgtls_write,
+ .buffered_read_data = (io_stream_predicate) pgtls_read_pending,
+ .destroy = (io_stream_destroy_func) pgtls_close
+};
static bool pq_init_ssl_lib = true;
static bool pq_init_crypto_lib = true;
@@ -141,8 +150,8 @@ pgtls_open_client(PGconn *conn)
return open_client_SSL(conn);
}
-ssize_t
-pgtls_read(PGconn *conn, void *ptr, size_t len)
+static ssize_t
+pgtls_read(IoStreamLayer * self, PGconn *conn, void *ptr, size_t len, bool buffered_only)
{
ssize_t n;
int result_errno = 0;
@@ -150,8 +159,20 @@ pgtls_read(PGconn *conn, void *ptr, size_t len)
int err;
unsigned long ecode;
+ if (buffered_only)
+ {
+ /*
+ * SSL_pending bytes are guaranteed to be available and readable
+ * without blocking
+ */
+ len = Min(len, SSL_pending(conn->ssl));
+ if (len == 0)
+ return 0;
+ }
+
rloop:
+
/*
* Prepare to call SSL_get_error() by clearing thread's OpenSSL error
* queue. In general, the current thread's error queue must be empty
@@ -257,14 +278,14 @@ rloop:
return n;
}
-bool
+static bool
pgtls_read_pending(PGconn *conn)
{
return SSL_pending(conn->ssl) > 0;
}
-ssize_t
-pgtls_write(PGconn *conn, const void *ptr, size_t len)
+static int
+pgtls_write(IoStreamLayer * self, PGconn *conn, const void *ptr, size_t len, size_t *bytes_written)
{
ssize_t n;
int result_errno = 0;
@@ -360,7 +381,17 @@ pgtls_write(PGconn *conn, const void *ptr, size_t len)
/* ensure we return the intended errno to caller */
SOCK_ERRNO_SET(result_errno);
- return n;
+
+ if (n >= 0)
+ {
+ *bytes_written = n;
+ return 0;
+ }
+ else
+ {
+ *bytes_written = 0;
+ return n;
+ }
}
char *
@@ -1211,7 +1242,8 @@ initialize_SSL(PGconn *conn)
*/
if (!(conn->ssl = SSL_new(SSL_context)) ||
!SSL_set_app_data(conn->ssl, conn) ||
- !my_SSL_set_fd(conn, conn->sock))
+ !my_SSL_set_fd(conn->ssl, conn->sock,
+ io_stream_add_layer(conn->io_stream, &pgtls_processor, conn)))
{
char *err = SSLerrmessage(ERR_get_error());
@@ -1613,7 +1645,7 @@ open_client_SSL(PGconn *conn)
return PGRES_POLLING_OK;
}
-void
+static void
pgtls_close(PGconn *conn)
{
bool destroy_needed = false;
@@ -1815,7 +1847,7 @@ PQsslAttribute(PGconn *conn, const char *attribute_name)
/*
* Private substitute BIO: this does the sending and receiving using
- * pqsecure_raw_write() and pqsecure_raw_read() instead, to allow those
+ * io_stream_next_write() and io_stream_next_read() instead, to allow those
* functions to disable SIGPIPE and give better error messages on I/O errors.
*
* These functions are closely modelled on the standard socket BIO in OpenSSL;
@@ -1830,7 +1862,7 @@ my_sock_read(BIO *h, char *buf, int size)
{
int res;
- res = pqsecure_raw_read((PGconn *) BIO_get_app_data(h), buf, size);
+ res = io_stream_next_read(BIO_get_app_data(h), buf, size, false);
BIO_clear_retry_flags(h);
if (res < 0)
{
@@ -1859,8 +1891,9 @@ static int
my_sock_write(BIO *h, const char *buf, int size)
{
int res;
+ size_t count;
- res = pqsecure_raw_write((PGconn *) BIO_get_app_data(h), buf, size);
+ res = io_stream_next_write(BIO_get_app_data(h), buf, size, &count);
BIO_clear_retry_flags(h);
if (res < 0)
{
@@ -1882,7 +1915,7 @@ my_sock_write(BIO *h, const char *buf, int size)
}
}
- return res;
+ return res == 0 ? count : res;
}
static BIO_METHOD *
@@ -1952,7 +1985,7 @@ err:
/* This should exactly match OpenSSL's SSL_set_fd except for using my BIO */
static int
-my_SSL_set_fd(PGconn *conn, int fd)
+my_SSL_set_fd(SSL *ssl, int fd, IoStreamLayer * layer)
{
int ret = 0;
BIO *bio;
@@ -1965,15 +1998,16 @@ my_SSL_set_fd(PGconn *conn, int fd)
goto err;
}
bio = BIO_new(bio_method);
+
if (bio == NULL)
{
SSLerr(SSL_F_SSL_SET_FD, ERR_R_BUF_LIB);
goto err;
}
- BIO_set_app_data(bio, conn);
+ BIO_set_app_data(bio, layer);
- SSL_set_bio(conn->ssl, bio, bio);
BIO_set_fd(bio, fd, BIO_NOCLOSE);
+ SSL_set_bio(ssl, bio, bio);
ret = 1;
err:
return ret;
diff --git a/src/interfaces/libpq/fe-secure.c b/src/interfaces/libpq/fe-secure.c
index b2430362a9..4b34de0220 100644
--- a/src/interfaces/libpq/fe-secure.c
+++ b/src/interfaces/libpq/fe-secure.c
@@ -45,55 +45,6 @@
#include "libpq-fe.h"
#include "libpq-int.h"
-/*
- * Macros to handle disabling and then restoring the state of SIGPIPE handling.
- * On Windows, these are all no-ops since there's no SIGPIPEs.
- */
-
-#ifndef WIN32
-
-#define SIGPIPE_MASKED(conn) ((conn)->sigpipe_so || (conn)->sigpipe_flag)
-
-struct sigpipe_info
-{
- sigset_t oldsigmask;
- bool sigpipe_pending;
- bool got_epipe;
-};
-
-#define DECLARE_SIGPIPE_INFO(spinfo) struct sigpipe_info spinfo
-
-#define DISABLE_SIGPIPE(conn, spinfo, failaction) \
- do { \
- (spinfo).got_epipe = false; \
- if (!SIGPIPE_MASKED(conn)) \
- { \
- if (pq_block_sigpipe(&(spinfo).oldsigmask, \
- &(spinfo).sigpipe_pending) < 0) \
- failaction; \
- } \
- } while (0)
-
-#define REMEMBER_EPIPE(spinfo, cond) \
- do { \
- if (cond) \
- (spinfo).got_epipe = true; \
- } while (0)
-
-#define RESTORE_SIGPIPE(conn, spinfo) \
- do { \
- if (!SIGPIPE_MASKED(conn)) \
- pq_reset_sigpipe(&(spinfo).oldsigmask, (spinfo).sigpipe_pending, \
- (spinfo).got_epipe); \
- } while (0)
-#else /* WIN32 */
-
-#define DECLARE_SIGPIPE_INFO(spinfo)
-#define DISABLE_SIGPIPE(conn, spinfo, failaction)
-#define REMEMBER_EPIPE(spinfo, cond)
-#define RESTORE_SIGPIPE(conn, spinfo)
-#endif /* WIN32 */
-
/* ------------------------------------------------------------ */
/* Procedures common to all secure sessions */
/* ------------------------------------------------------------ */
@@ -160,281 +111,6 @@ pqsecure_open_client(PGconn *conn)
#endif
}
-/*
- * Close secure session.
- */
-void
-pqsecure_close(PGconn *conn)
-{
-#ifdef USE_SSL
- pgtls_close(conn);
-#endif
-}
-
-/*
- * Read data from a secure connection.
- *
- * On failure, this function is responsible for appending a suitable message
- * to conn->errorMessage. The caller must still inspect errno, but only
- * to determine whether to continue/retry after error.
- */
-ssize_t
-pqsecure_read(PGconn *conn, void *ptr, size_t len)
-{
- ssize_t n;
-
-#ifdef USE_SSL
- if (conn->ssl_in_use)
- {
- n = pgtls_read(conn, ptr, len);
- }
- else
-#endif
-#ifdef ENABLE_GSS
- if (conn->gssenc)
- {
- n = pg_GSS_read(conn, ptr, len);
- }
- else
-#endif
- {
- n = pqsecure_raw_read(conn, ptr, len);
- }
-
- return n;
-}
-
-ssize_t
-pqsecure_raw_read(PGconn *conn, void *ptr, size_t len)
-{
- ssize_t n;
- int result_errno = 0;
- char sebuf[PG_STRERROR_R_BUFLEN];
-
- SOCK_ERRNO_SET(0);
-
- n = recv(conn->sock, ptr, len, 0);
-
- if (n < 0)
- {
- result_errno = SOCK_ERRNO;
-
- /* Set error message if appropriate */
- switch (result_errno)
- {
-#ifdef EAGAIN
- case EAGAIN:
-#endif
-#if defined(EWOULDBLOCK) && (!defined(EAGAIN) || (EWOULDBLOCK != EAGAIN))
- case EWOULDBLOCK:
-#endif
- case EINTR:
- /* no error message, caller is expected to retry */
- break;
-
- case EPIPE:
- case ECONNRESET:
- libpq_append_conn_error(conn, "server closed the connection unexpectedly\n"
- "\tThis probably means the server terminated abnormally\n"
- "\tbefore or while processing the request.");
- break;
-
- case 0:
- /* If errno didn't get set, treat it as regular EOF */
- n = 0;
- break;
-
- default:
- libpq_append_conn_error(conn, "could not receive data from server: %s",
- SOCK_STRERROR(result_errno,
- sebuf, sizeof(sebuf)));
- break;
- }
- }
-
- /* ensure we return the intended errno to caller */
- SOCK_ERRNO_SET(result_errno);
-
- return n;
-}
-
-/*
- * Write data to a secure connection.
- *
- * Returns the number of bytes written, or a negative value (with errno
- * set) upon failure. The write count could be less than requested.
- *
- * Note that socket-level hard failures are masked from the caller,
- * instead setting conn->write_failed and storing an error message
- * in conn->write_err_msg; see pqsecure_raw_write. This allows us to
- * postpone reporting of write failures until we're sure no error
- * message is available from the server.
- *
- * However, errors detected in the SSL or GSS management level are reported
- * via a negative result, with message appended to conn->errorMessage.
- * It's frequently unclear whether such errors should be considered read or
- * write errors, so we don't attempt to postpone reporting them.
- *
- * The caller must still inspect errno upon failure, but only to determine
- * whether to continue/retry; a message has been saved someplace in any case.
- */
-ssize_t
-pqsecure_write(PGconn *conn, const void *ptr, size_t len)
-{
- ssize_t n;
-
-#ifdef USE_SSL
- if (conn->ssl_in_use)
- {
- n = pgtls_write(conn, ptr, len);
- }
- else
-#endif
-#ifdef ENABLE_GSS
- if (conn->gssenc)
- {
- n = pg_GSS_write(conn, ptr, len);
- }
- else
-#endif
- {
- n = pqsecure_raw_write(conn, ptr, len);
- }
-
- return n;
-}
-
-/*
- * Low-level implementation of pqsecure_write.
- *
- * This is used directly for an unencrypted connection. For encrypted
- * connections, this does the physical I/O on behalf of pgtls_write or
- * pg_GSS_write.
- *
- * This function reports failure (i.e., returns a negative result) only
- * for retryable errors such as EINTR. Looping for such cases is to be
- * handled at some outer level, maybe all the way up to the application.
- * For hard failures, we set conn->write_failed and store an error message
- * in conn->write_err_msg, but then claim to have written the data anyway.
- * This is because we don't want to report write failures so long as there
- * is a possibility of reading from the server and getting an error message
- * that could explain why the connection dropped. Many TCP stacks have
- * race conditions such that a write failure may or may not be reported
- * before all incoming data has been read.
- *
- * Note that this error behavior happens below the SSL management level when
- * we are using SSL. That's because at least some versions of OpenSSL are
- * too quick to report a write failure when there's still a possibility to
- * get a more useful error from the server.
- */
-ssize_t
-pqsecure_raw_write(PGconn *conn, const void *ptr, size_t len)
-{
- ssize_t n;
- int flags = 0;
- int result_errno = 0;
- char msgbuf[1024];
- char sebuf[PG_STRERROR_R_BUFLEN];
-
- DECLARE_SIGPIPE_INFO(spinfo);
-
- /*
- * If we already had a write failure, we will never again try to send data
- * on that connection. Even if the kernel would let us, we've probably
- * lost message boundary sync with the server. conn->write_failed
- * therefore persists until the connection is reset, and we just discard
- * all data presented to be written.
- */
- if (conn->write_failed)
- return len;
-
-#ifdef MSG_NOSIGNAL
- if (conn->sigpipe_flag)
- flags |= MSG_NOSIGNAL;
-
-retry_masked:
-#endif /* MSG_NOSIGNAL */
-
- DISABLE_SIGPIPE(conn, spinfo, return -1);
-
- n = send(conn->sock, ptr, len, flags);
-
- if (n < 0)
- {
- result_errno = SOCK_ERRNO;
-
- /*
- * If we see an EINVAL, it may be because MSG_NOSIGNAL isn't available
- * on this machine. So, clear sigpipe_flag so we don't try the flag
- * again, and retry the send().
- */
-#ifdef MSG_NOSIGNAL
- if (flags != 0 && result_errno == EINVAL)
- {
- conn->sigpipe_flag = false;
- flags = 0;
- goto retry_masked;
- }
-#endif /* MSG_NOSIGNAL */
-
- /* Set error message if appropriate */
- switch (result_errno)
- {
-#ifdef EAGAIN
- case EAGAIN:
-#endif
-#if defined(EWOULDBLOCK) && (!defined(EAGAIN) || (EWOULDBLOCK != EAGAIN))
- case EWOULDBLOCK:
-#endif
- case EINTR:
- /* no error message, caller is expected to retry */
- break;
-
- case EPIPE:
- /* Set flag for EPIPE */
- REMEMBER_EPIPE(spinfo, true);
-
- /* FALL THRU */
-
- case ECONNRESET:
- conn->write_failed = true;
- /* Store error message in conn->write_err_msg, if possible */
- /* (strdup failure is OK, we'll cope later) */
- snprintf(msgbuf, sizeof(msgbuf),
- libpq_gettext("server closed the connection unexpectedly\n"
- "\tThis probably means the server terminated abnormally\n"
- "\tbefore or while processing the request."));
- /* keep newline out of translated string */
- strlcat(msgbuf, "\n", sizeof(msgbuf));
- conn->write_err_msg = strdup(msgbuf);
- /* Now claim the write succeeded */
- n = len;
- break;
-
- default:
- conn->write_failed = true;
- /* Store error message in conn->write_err_msg, if possible */
- /* (strdup failure is OK, we'll cope later) */
- snprintf(msgbuf, sizeof(msgbuf),
- libpq_gettext("could not send data to server: %s"),
- SOCK_STRERROR(result_errno,
- sebuf, sizeof(sebuf)));
- /* keep newline out of translated string */
- strlcat(msgbuf, "\n", sizeof(msgbuf));
- conn->write_err_msg = strdup(msgbuf);
- /* Now claim the write succeeded */
- n = len;
- break;
- }
- }
-
- RESTORE_SIGPIPE(conn, spinfo);
-
- /* ensure we return the intended errno to caller */
- SOCK_ERRNO_SET(result_errno);
-
- return n;
-}
/* Dummy versions of SSL info functions, when built without SSL support */
#ifndef USE_SSL
@@ -507,90 +183,3 @@ PQgssEncInUse(PGconn *conn)
}
#endif /* ENABLE_GSS */
-
-
-#if !defined(WIN32)
-
-/*
- * Block SIGPIPE for this thread. This prevents send()/write() from exiting
- * the application.
- */
-int
-pq_block_sigpipe(sigset_t *osigset, bool *sigpipe_pending)
-{
- sigset_t sigpipe_sigset;
- sigset_t sigset;
-
- sigemptyset(&sigpipe_sigset);
- sigaddset(&sigpipe_sigset, SIGPIPE);
-
- /* Block SIGPIPE and save previous mask for later reset */
- SOCK_ERRNO_SET(pthread_sigmask(SIG_BLOCK, &sigpipe_sigset, osigset));
- if (SOCK_ERRNO)
- return -1;
-
- /* We can have a pending SIGPIPE only if it was blocked before */
- if (sigismember(osigset, SIGPIPE))
- {
- /* Is there a pending SIGPIPE? */
- if (sigpending(&sigset) != 0)
- return -1;
-
- if (sigismember(&sigset, SIGPIPE))
- *sigpipe_pending = true;
- else
- *sigpipe_pending = false;
- }
- else
- *sigpipe_pending = false;
-
- return 0;
-}
-
-/*
- * Discard any pending SIGPIPE and reset the signal mask.
- *
- * Note: we are effectively assuming here that the C library doesn't queue
- * up multiple SIGPIPE events. If it did, then we'd accidentally leave
- * ours in the queue when an event was already pending and we got another.
- * As long as it doesn't queue multiple events, we're OK because the caller
- * can't tell the difference.
- *
- * The caller should say got_epipe = false if it is certain that it
- * didn't get an EPIPE error; in that case we'll skip the clear operation
- * and things are definitely OK, queuing or no. If it got one or might have
- * gotten one, pass got_epipe = true.
- *
- * We do not want this to change errno, since if it did that could lose
- * the error code from a preceding send(). We essentially assume that if
- * we were able to do pq_block_sigpipe(), this can't fail.
- */
-void
-pq_reset_sigpipe(sigset_t *osigset, bool sigpipe_pending, bool got_epipe)
-{
- int save_errno = SOCK_ERRNO;
- int signo;
- sigset_t sigset;
-
- /* Clear SIGPIPE only if none was pending */
- if (got_epipe && !sigpipe_pending)
- {
- if (sigpending(&sigset) == 0 &&
- sigismember(&sigset, SIGPIPE))
- {
- sigset_t sigpipe_sigset;
-
- sigemptyset(&sigpipe_sigset);
- sigaddset(&sigpipe_sigset, SIGPIPE);
-
- sigwait(&sigpipe_sigset, &signo);
- }
- }
-
- /* Restore saved block mask */
- pthread_sigmask(SIG_SETMASK, osigset, NULL);
-
- SOCK_ERRNO_SET(save_errno);
-}
-
-#endif /* !WIN32 */
diff --git a/src/interfaces/libpq/libpq-int.h b/src/interfaces/libpq/libpq-int.h
index 7888199b0d..1314663213 100644
--- a/src/interfaces/libpq/libpq-int.h
+++ b/src/interfaces/libpq/libpq-int.h
@@ -81,6 +81,7 @@ typedef struct
#endif
#endif /* USE_OPENSSL */
+#include "common/io_stream.h"
#include "common/pg_prng.h"
/*
@@ -456,6 +457,7 @@ struct pg_conn
PGcmdQueueEntry *cmd_queue_recycle;
/* Connection data */
+ IoStream *io_stream;
pgsocket sock; /* FD for socket, PGINVALID_SOCKET if
* unconnected */
SockAddr laddr; /* Local address */
@@ -753,11 +755,6 @@ extern int pqWriteReady(PGconn *conn);
extern int pqsecure_initialize(PGconn *, bool, bool);
extern PostgresPollingStatusType pqsecure_open_client(PGconn *);
-extern void pqsecure_close(PGconn *);
-extern ssize_t pqsecure_read(PGconn *, void *ptr, size_t len);
-extern ssize_t pqsecure_write(PGconn *, const void *ptr, size_t len);
-extern ssize_t pqsecure_raw_read(PGconn *, void *ptr, size_t len);
-extern ssize_t pqsecure_raw_write(PGconn *, const void *ptr, size_t len);
#if !defined(WIN32)
extern int pq_block_sigpipe(sigset_t *osigset, bool *sigpipe_pending);
@@ -793,34 +790,6 @@ extern int pgtls_init(PGconn *conn, bool do_ssl, bool do_crypto);
*/
extern PostgresPollingStatusType pgtls_open_client(PGconn *conn);
-/*
- * Close SSL connection.
- */
-extern void pgtls_close(PGconn *conn);
-
-/*
- * Read data from a secure connection.
- *
- * On failure, this function is responsible for appending a suitable message
- * to conn->errorMessage. The caller must still inspect errno, but only
- * to determine whether to continue/retry after error.
- */
-extern ssize_t pgtls_read(PGconn *conn, void *ptr, size_t len);
-
-/*
- * Is there unread data waiting in the SSL read buffer?
- */
-extern bool pgtls_read_pending(PGconn *conn);
-
-/*
- * Write data to a secure connection.
- *
- * On failure, this function is responsible for appending a suitable message
- * to conn->errorMessage. The caller must still inspect errno, but only
- * to determine whether to continue/retry after error.
- */
-extern ssize_t pgtls_write(PGconn *conn, const void *ptr, size_t len);
-
/*
* Get the hash of the server certificate, for SCRAM channel binding type
* tls-server-end-point.
@@ -851,13 +820,6 @@ extern int pgtls_verify_peer_name_matches_certificate_guts(PGconn *conn,
* Establish a GSSAPI-encrypted connection.
*/
extern PostgresPollingStatusType pqsecure_open_gss(PGconn *conn);
-
-/*
- * Read and write functions for GSSAPI-encrypted connections, with internal
- * buffering to handle nonblocking sockets.
- */
-extern ssize_t pg_GSS_write(PGconn *conn, const void *ptr, size_t len);
-extern ssize_t pg_GSS_read(PGconn *conn, void *ptr, size_t len);
#endif
/* === in fe-trace.c === */
--
2.42.0
[application/octet-stream] v3-0002-Add-protocol-layer-compression-to-libpq.patch (132.0K, ../../CACzsqT5-7xfbz+Si35TBYHzerNX3XJVzAUH9AewQ+Pp13fYBoQ@mail.gmail.com/3-v3-0002-Add-protocol-layer-compression-to-libpq.patch)
download | inline diff:
From 77225482632a77dcadfd1293dd4776334c151964 Mon Sep 17 00:00:00 2001
From: Jacob Burroughs <[email protected]>
Date: Fri, 15 Dec 2023 12:21:08 -0600
Subject: [PATCH v3 2/5] Add protocol-layer compression to libpq
Adds libpq_compression and libpq_fe_compression GUCs to coordinate
algorithms between frontend and backend. Adds CompressedMessage
message to send compressed data and select a particular negotiated
protocol respectively. Supported compression algorithms are zlib
(gzip), lz4, and zstd.
fix
---
contrib/postgres_fdw/connection.c | 26 +-
doc/src/sgml/config.sgml | 38 +
doc/src/sgml/libpq.sgml | 62 +
doc/src/sgml/protocol.sgml | 58 +
meson.build | 4 +-
src/backend/backup/basebackup.c | 2 +-
src/backend/libpq/Makefile | 1 +
src/backend/libpq/be-secure-openssl.c | 2 +-
src/backend/libpq/compression.c | 127 +++
src/backend/libpq/meson.build | 1 +
src/backend/libpq/pqcomm.c | 87 +-
src/backend/postmaster/postmaster.c | 10 +-
.../libpqwalreceiver/libpqwalreceiver.c | 15 +-
src/backend/utils/misc/guc_tables.c | 12 +
src/backend/utils/misc/postgresql.conf.sample | 3 +
src/bin/pg_basebackup/pg_basebackup.c | 2 +-
src/bin/pg_basebackup/pg_receivewal.c | 2 +-
src/bin/pg_dump/pg_dump.c | 11 +-
src/bin/pgbench/pgbench.c | 17 +-
src/bin/psql/command.c | 18 +
src/common/Makefile | 4 +-
src/common/compression.c | 229 +++-
src/common/io_stream.c | 13 +
src/common/meson.build | 2 +
src/common/z_stream.c | 995 ++++++++++++++++
src/common/zpq_stream.c | 1013 +++++++++++++++++
src/include/common/compression.h | 18 +-
src/include/common/io_stream.h | 17 +-
src/include/common/z_stream.h | 98 ++
src/include/common/zpq_stream.h | 91 ++
src/include/libpq/compression.h | 30 +
src/include/libpq/libpq-be-fe-helpers.h | 13 +-
src/include/libpq/libpq-be.h | 3 +
src/include/libpq/libpq.h | 1 +
src/include/libpq/protocol.h | 2 +-
src/include/utils/guc_hooks.h | 2 +
src/interfaces/libpq/exports.txt | 3 +
src/interfaces/libpq/fe-connect.c | 129 ++-
src/interfaces/libpq/fe-exec.c | 15 +
src/interfaces/libpq/fe-misc.c | 47 +-
src/interfaces/libpq/fe-protocol3.c | 92 +-
src/interfaces/libpq/fe-secure-openssl.c | 2 +-
src/interfaces/libpq/libpq-fe.h | 4 +
src/interfaces/libpq/libpq-int.h | 18 +-
44 files changed, 3175 insertions(+), 164 deletions(-)
create mode 100644 src/backend/libpq/compression.c
create mode 100644 src/common/z_stream.c
create mode 100644 src/common/zpq_stream.c
create mode 100644 src/include/common/z_stream.h
create mode 100644 src/include/common/zpq_stream.h
create mode 100644 src/include/libpq/compression.h
diff --git a/contrib/postgres_fdw/connection.c b/contrib/postgres_fdw/connection.c
index 5800c6a9fb..82c7b17b58 100644
--- a/contrib/postgres_fdw/connection.c
+++ b/contrib/postgres_fdw/connection.c
@@ -872,11 +872,14 @@ pgfdw_get_result(PGconn *conn, const char *query)
pgfdw_we_get_result = WaitEventExtensionNew("PostgresFdwGetResult");
/* Sleep until there's something to do */
- wc = WaitLatchOrSocket(MyLatch,
- WL_LATCH_SET | WL_SOCKET_READABLE |
- WL_EXIT_ON_PM_DEATH,
- PQsocket(conn),
- -1L, pgfdw_we_get_result);
+ if (PQreadPending(conn))
+ wc = WL_SOCKET_READABLE;
+ else
+ wc = WaitLatchOrSocket(MyLatch,
+ WL_LATCH_SET | WL_SOCKET_READABLE |
+ WL_EXIT_ON_PM_DEATH,
+ PQsocket(conn),
+ -1L, pgfdw_we_get_result);
ResetLatch(MyLatch);
CHECK_FOR_INTERRUPTS();
@@ -1580,11 +1583,14 @@ pgfdw_get_cleanup_result(PGconn *conn, TimestampTz endtime, PGresult **result,
pgfdw_we_cleanup_result = WaitEventExtensionNew("PostgresFdwCleanupResult");
/* Sleep until there's something to do */
- wc = WaitLatchOrSocket(MyLatch,
- WL_LATCH_SET | WL_SOCKET_READABLE |
- WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
- PQsocket(conn),
- cur_timeout, pgfdw_we_cleanup_result);
+ if (PQreadPending(conn))
+ wc = WL_SOCKET_READABLE;
+ else
+ wc = WaitLatchOrSocket(MyLatch,
+ WL_LATCH_SET | WL_SOCKET_READABLE |
+ WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
+ PQsocket(conn),
+ cur_timeout, pgfdw_we_cleanup_result);
ResetLatch(MyLatch);
CHECK_FOR_INTERRUPTS();
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index f323bba018..fa9a7bd418 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -1066,6 +1066,44 @@ include_dir 'conf.d'
</listitem>
</varlistentry>
+ <varlistentry id="guc-libpq-compression" xreflabel="libpq_compression">
+ <term><varname>libpq_compression</varname> (<type>string</type>)
+ <indexterm>
+ <primary><varname>libpq_compression</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ This parameter controls the available client-server traffic compression methods.
+ It allows rejecting compression requests even if it is supported by the server (for example, due to security, or CPU consumption).
+ The default is <literal>off</literal>, which means that no compression methods are allowed.
+ The value <literal>on</literal> means all supported compression methods are allowed.
+ For more precise control, a list of the allowed compression methods can be specified.
+ For example, to allow only <literal>lz4</literal> and <literal>gzip</literal>, set the setting to <literal>lz4;gzip</literal>.
+ The server will choose the first algorithm from the list also supported by a given client.
+ </para>
+ <para>
+ The compression methods can be specified as <literal>gzip</literal>,
+ <literal>lz4</literal>, or <literal>zstd</literal>.
+ A compression detail string can optionally be specified.
+ If the detail string is an integer, it specifies the compression
+ level. Otherwise, it should be a comma-separated list of items,
+ each of the form
+ <replaceable>keyword</replaceable> or
+ <replaceable>keyword=value</replaceable>.
+ Currently, the supported keywords are <literal>level</literal>,
+ <literal>compress</literal>, and <literal>decompress</literal>.
+ The detail string cannot be used when the compression method
+ is specified as a plain integer.
+ <literal>compress</literal> and <literal>decompress</literal>
+ both default to true, but can be set to false to allow using an algorithm for only compression or
+ only for decompression, which is most useful when enabling asymmetric compression.
+ e.g. to enable compression for client-to-server traffic but not server-to-client traffic,
+ you can set the parameter to <literal>"gzip:compress=false"</literal>.
+ </para>
+ </listitem>
+ </varlistentry>
+
</variablelist>
</sect2>
diff --git a/doc/src/sgml/libpq.sgml b/doc/src/sgml/libpq.sgml
index ed88ac001a..bf39203174 100644
--- a/doc/src/sgml/libpq.sgml
+++ b/doc/src/sgml/libpq.sgml
@@ -1354,6 +1354,68 @@ postgresql://%2Fvar%2Flib%2Fpostgresql/dbname
</listitem>
</varlistentry>
+ <varlistentry id="libpq-connect-compression" xreflabel="compression">
+ <term><literal>compression</literal></term>
+ <listitem>
+ <para>
+ Request compression of libpq traffic. The client sends a request with
+ a list of compression algorithms. Compression can be requested by a client
+ by including the <literal>"compression"</literal> option in its connection string.
+ This can either be a boolean value to enable or disable compression
+ (<literal>"true"</literal>/<literal>"false"</literal>,
+ <literal>"on"</literal>/<literal>"off"</literal>,
+ <literal>"yes"</literal>/<literal>"no"</literal>,
+ <literal>"1"</literal>/<literal>"0"</literal>),
+ or an explicit list of semicolon-separated compression algorithms (<literal>"gzip;lz4:2"</literal>).
+ The default value is <literal>"off"</literal>.
+ If compression is enabled but an algorithm is not explicitly specified,
+ the client library sends its full list of supported algorithms.
+ The server sends its list of supported parameters in the <varname>libpq_compression</varname>
+ parameter.
+ Both the client and server will prefer the first algorithm in their list that is supported by
+ the other side.
+ </para>
+ <para>
+ The compression methods can be specified as <literal>gzip</literal>,
+ <literal>lz4</literal>, or <literal>zstd</literal>.
+ A compression detail string can optionally be specified.
+ If the detail string is an integer, it specifies the compression
+ level. Otherwise, it should be a comma-separated list of items,
+ each of the form
+ <replaceable>keyword</replaceable> or
+ <replaceable>keyword=value</replaceable>.
+ Currently, the supported keywords are <literal>level</literal>,
+ <literal>compress</literal>, and <literal>decompress</literal>.
+ The detail string cannot be used when the compression method
+ is specified as a plain integer.
+ <literal>compress</literal> and <literal>decompress</literal>
+ both default to true, but can be set to false to allow using an algorithm for only compression or
+ only for decompression, which is most useful when enabling asymmetric compression.
+ e.g. to enable compression for server-to-client traffic but not client-to-server traffic,
+ you can set the parameter to <literal>"gzip:compress=false"</literal>.
+ </para>
+ <para>
+ After receiving a startup packet with <varname>_pq_.libpq_compression</varname> set, the
+ server can send CompressedData messages referencing any of the specified algorithms, for
+ server-to-client traffic compression.
+ After receiving a parameter status message with <varname>libpq_compression</varname> set,
+ the client can send CompressedData messages referencing any of the specified algorithms for
+ client-to-server traffic compression. (Note that if the client has not requested the
+ <varname>_pq_.libpq_compression</varname> protocol extension in the startup packet, the
+ server may reject all CompressedData messages even if <varname>libpq_compression</varname>
+ is non-empty.)
+ </para>
+ <para>
+ Support for compression algorithms must be enabled when the server is compiled.
+ Currently, three algorithms are supported: gzip (default), lz4 (if Postgres was
+ configured with --with-lz4 option), and zstd (if Postgres was configured with
+ --with-zstd option). In all cases, streaming mode is used.
+ Please note that using compression together with SSL may expose extra vulnerabilities:
+ <ulink url="https://en.wikipedia.org/wiki/CRIME">CRIME</ulink>
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry id="libpq-connect-client-encoding" xreflabel="client_encoding">
<term><literal>client_encoding</literal></term>
<listitem>
diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index 6c3e8a631d..8b7bf3bc96 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -92,6 +92,15 @@
such as <command>COPY</command>.
</para>
+ <para>
+ The protocol supports compressing data to reduce traffic and speed-up client-server interaction.
+ Compression is especially useful for importing/exporting data to/from the database using the <literal>COPY</literal> command
+ and for replication (both physical and logical). Compression can also reduce the server's response time
+ for queries returning a large amount of data (for example, JSON, BLOBs, text, ...).
+ Currently, three algorithms are supported: DEFLATE (if PostgreSQL was built with zlib support),
+ LZ4 (if PostgreSQL was built with lz4 support), and ZStandard (if PostgresSQL was built with zstd support).
+ </para>
+
<sect2 id="protocol-message-concepts">
<title>Messaging Overview</title>
@@ -4216,6 +4225,55 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
</listitem>
</varlistentry>
+ <varlistentry id="protocol-message-formats-CompressedData">
+ <term>CompressedData (F & B)</term>
+ <listitem>
+ <para>
+
+ <variablelist>
+ <varlistentry>
+ <term>Byte1('z')</term>
+ <listitem>
+ <para>
+ Identifies the message as compressed data.
+ </para>
+ </listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>Int32</term>
+ <listitem>
+ <para>
+ Length of message contents in bytes, including self.
+ </para>
+ </listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>Int8</term>
+ <listitem>
+ <para>
+ Selected compression algorithm, as specified in the pg_compress_algorithm enum, or
+ <literal>0</literal> to continue using the decompression stream started in a previous
+ <literal>CompressedData</literal> message. When processing a message with a non-zero
+ value for this parameter, the client must first restart the decompression stream.
+ </para>
+ </listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>
+ Byte<replaceable>n</replaceable>
+ </term>
+ <listitem>
+ <para>
+ Compressed message data.
+ </para>
+ </listitem>
+ </varlistentry>
+ </variablelist>
+
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry id="protocol-message-formats-CopyData">
<term>CopyData (F & B)</term>
<listitem>
diff --git a/meson.build b/meson.build
index 21abd7da85..80fe2d1316 100644
--- a/meson.build
+++ b/meson.build
@@ -2793,14 +2793,14 @@ frontend_common_code = declare_dependency(
compile_args: ['-DFRONTEND'],
include_directories: [postgres_inc],
sources: generated_headers,
- dependencies: [os_deps, zlib, zstd],
+ dependencies: [os_deps, lz4, zlib, zstd],
)
backend_common_code = declare_dependency(
compile_args: ['-DBUILDING_DLL'],
include_directories: [postgres_inc],
sources: generated_headers,
- dependencies: [os_deps, zlib, zstd],
+ dependencies: [os_deps, lz4, zlib, zstd],
)
subdir('src/common')
diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c
index 84246739ae..983662f4af 100644
--- a/src/backend/backup/basebackup.c
+++ b/src/backend/backup/basebackup.c
@@ -968,7 +968,7 @@ parse_basebackup_options(List *options, basebackup_options *opt)
char *error_detail;
parse_compress_specification(opt->compression, compression_detail_str,
- &opt->compression_specification);
+ &opt->compression_specification, PG_COMPRESSION_OPTION_WORKERS | PG_COMPRESSION_OPTION_LONG_DISTANCE);
error_detail =
validate_compress_specification(&opt->compression_specification);
if (error_detail != NULL)
diff --git a/src/backend/libpq/Makefile b/src/backend/libpq/Makefile
index 6d385fd6a4..f4b09c09c3 100644
--- a/src/backend/libpq/Makefile
+++ b/src/backend/libpq/Makefile
@@ -21,6 +21,7 @@ OBJS = \
be-fsstubs.o \
be-secure-common.o \
be-secure.o \
+ compression.o \
crypt.o \
hba.o \
ifaddr.o \
diff --git a/src/backend/libpq/be-secure-openssl.c b/src/backend/libpq/be-secure-openssl.c
index 5c67fd46aa..0125b50b95 100644
--- a/src/backend/libpq/be-secure-openssl.c
+++ b/src/backend/libpq/be-secure-openssl.c
@@ -78,7 +78,7 @@ static void be_tls_close(Port *port);
IoStreamProcessor be_tls_processor = {
.read = (io_stream_read_func) be_tls_read,
.write = (io_stream_write_func) be_tls_write,
- .destroy = (io_stream_destroy_func) be_tls_close
+ .destroy = (io_stream_consumer) be_tls_close
};
static char *X509_NAME_to_cstring(X509_NAME *name);
diff --git a/src/backend/libpq/compression.c b/src/backend/libpq/compression.c
new file mode 100644
index 0000000000..bd0f97b32a
--- /dev/null
+++ b/src/backend/libpq/compression.c
@@ -0,0 +1,127 @@
+/*-------------------------------------------------------------------------
+*
+ * compression.c
+ * Functions and variables to support backend configuration of libpq
+ *
+ *
+ * Copyright (c) 2023, PostgreSQL Global Development Group
+ *
+ * src/backend/libpq/compression.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+#include "miscadmin.h"
+#include "libpq/libpq-be.h"
+#include "libpq/compression.h"
+#include "utils/guc_hooks.h"
+
+/* GUC variable containing the allowed compression algorithms list (separated by semicolon) */
+char *libpq_compress_algorithms = "off";
+pg_compress_specification libpq_compressors[COMPRESSION_ALGORITHM_COUNT];
+size_t libpq_n_compressors = 0;
+
+bool
+check_libpq_compression(char **newval, void **extra, GucSource source)
+{
+ pg_compress_specification compressors[COMPRESSION_ALGORITHM_COUNT];
+ size_t n_compressors;
+ char *serialized_compressors;
+
+ if (zpq_parse_compression_setting(*newval, compressors, &n_compressors) == -1)
+ {
+ GUC_check_errdetail("Cannot parse the libpq_compression setting.");
+ return false;
+ }
+
+ if (n_compressors > 0)
+ {
+ guc_free(*newval);
+ serialized_compressors = zpq_serialize_compressors(compressors, n_compressors);
+ *newval = guc_strdup(ERROR, serialized_compressors);
+ pfree(serialized_compressors);
+ }
+ else
+ {
+ guc_free(*newval);
+ *newval = guc_strdup(ERROR, "");
+ }
+ return true;
+}
+
+void
+assign_libpq_compression(const char *newval, void *extra)
+{
+ if (strlen(newval) == 0)
+ {
+ libpq_n_compressors = 0;
+ return;
+ }
+ zpq_parse_compression_setting(newval, libpq_compressors, &libpq_n_compressors);
+}
+
+void
+configure_libpq_compression(Port *port, const char *newval)
+{
+ pg_compress_specification fe_compressors[COMPRESSION_ALGORITHM_COUNT];
+ size_t n_fe_compressors;
+
+ Assert(!port->zpq_stream);
+
+ if (libpq_n_compressors == 0)
+ {
+ return;
+ }
+
+ /* Init compression */
+ port->zpq_stream = zpq_create(libpq_compressors, libpq_n_compressors, MyProcPort->io_stream);
+ if (!port->zpq_stream)
+ {
+ ereport(FATAL,
+ errcode(ERRCODE_INTERNAL_ERROR),
+ errmsg("failed to initialize the compression stream"));
+ }
+
+ if (strlen(newval) == 0)
+ {
+ return;
+ }
+
+ if (!zpq_deserialize_compressors(newval, fe_compressors, &n_fe_compressors))
+ {
+ ereport(FATAL,
+ errcode(ERRCODE_INTERNAL_ERROR),
+ errmsg("Cannot parse the _pq_.libpq_compression setting."));
+ }
+
+ if (MyProcPort && MyProcPort->zpq_stream)
+ {
+ pg_compress_algorithm algorithms[COMPRESSION_ALGORITHM_COUNT];
+ size_t n_algorithms = 0;
+
+ if (n_fe_compressors == 0)
+ {
+ return;
+ }
+
+ /*
+ * Intersect client and server compressors to determine the final list
+ * of the supported compressors. O(N^2) is negligible because of a
+ * small number of the compression methods.
+ */
+ for (size_t i = 0; i < libpq_n_compressors; i++)
+ {
+ for (size_t j = 0; j < n_fe_compressors; j++)
+ {
+ if (libpq_compressors[i].algorithm == fe_compressors[j].algorithm && libpq_compressors[i].compress && fe_compressors[j].decompress)
+ {
+ algorithms[n_algorithms] = libpq_compressors[i].algorithm;
+ n_algorithms += 1;
+ break;
+ }
+ }
+ }
+
+ zpq_enable_compression(MyProcPort->zpq_stream, algorithms, n_algorithms);
+ }
+}
diff --git a/src/backend/libpq/meson.build b/src/backend/libpq/meson.build
index 74a226c2bd..83a21a7566 100644
--- a/src/backend/libpq/meson.build
+++ b/src/backend/libpq/meson.build
@@ -7,6 +7,7 @@ backend_sources += files(
'be-fsstubs.c',
'be-secure-common.c',
'be-secure.c',
+ 'compression.c',
'crypt.c',
'hba.c',
'ifaddr.c',
diff --git a/src/backend/libpq/pqcomm.c b/src/backend/libpq/pqcomm.c
index 030686cc3b..ed4aad57e6 100644
--- a/src/backend/libpq/pqcomm.c
+++ b/src/backend/libpq/pqcomm.c
@@ -61,6 +61,7 @@
#include <signal.h>
#include <fcntl.h>
#include <grp.h>
+#include <pgstat.h>
#include <unistd.h>
#include <sys/file.h>
#include <sys/socket.h>
@@ -76,7 +77,9 @@
#include "common/ip.h"
#include "common/io_stream.h"
+#include "common/zpq_stream.h"
#include "libpq/libpq.h"
+#include "libpq/pqformat.h"
#include "miscadmin.h"
#include "port/pg_bswap.h"
#include "storage/ipc.h"
@@ -84,6 +87,7 @@
#include "utils/guc_hooks.h"
#include "utils/memutils.h"
#include "utils/wait_event.h"
+#include "utils/builtins.h"
/*
* Cope with the various platform-specific ways to spell TCP keepalive socket
@@ -1133,11 +1137,12 @@ retry:
/* --------------------------------
* pq_recvbuf - load some bytes into the input buffer
*
- * returns 0 if OK, EOF if trouble
+ * nowait parameter toggles non-blocking mode.
+ * returns number of read bytes, EOF if trouble
* --------------------------------
*/
static int
-pq_recvbuf(void)
+pq_recvbuf(bool nowait)
{
if (PqRecvPointer > 0)
{
@@ -1153,8 +1158,8 @@ pq_recvbuf(void)
PqRecvLength = PqRecvPointer = 0;
}
- /* Ensure that we're in blocking mode */
- socket_set_nonblocking(false);
+ /* Ensure that we're in the appropriate mode */
+ socket_set_nonblocking(nowait);
/* Can fill buffer from PqRecvLength and upwards */
for (;;)
@@ -1168,9 +1173,23 @@ pq_recvbuf(void)
if (r < 0)
{
+ if (r == ZS_DECOMPRESS_ERROR)
+ {
+ char const *msg = zpq_decompress_error(MyProcPort->zpq_stream);
+
+ if (msg == NULL)
+ msg = "end of stream";
+ ereport(COMMERROR,
+ (errcode_for_socket_access(),
+ errmsg("failed to decompress data: %s", msg)));
+ return EOF;
+ }
if (errno == EINTR)
continue; /* Ok if interrupted */
+ if (nowait && (errno == EAGAIN || errno == EWOULDBLOCK))
+ return 0;
+
/*
* Careful: an ereport() that tries to write to the client would
* cause recursion to here, leading to stack overflow and core
@@ -1194,7 +1213,7 @@ pq_recvbuf(void)
}
/* r contains number of bytes read, so just incr length */
PqRecvLength += r;
- return 0;
+ return r;
}
}
@@ -1209,7 +1228,8 @@ pq_getbyte(void)
while (PqRecvPointer >= PqRecvLength)
{
- if (pq_recvbuf()) /* If nothing in buffer, then recv some */
+ if (pq_recvbuf(false) == EOF) /* If nothing in buffer, then recv
+ * some */
return EOF; /* Failed to recv data */
}
return (unsigned char) PqRecvBuffer[PqRecvPointer++];
@@ -1228,7 +1248,8 @@ pq_peekbyte(void)
while (PqRecvPointer >= PqRecvLength)
{
- if (pq_recvbuf()) /* If nothing in buffer, then recv some */
+ if (pq_recvbuf(false) == EOF) /* If nothing in buffer, then recv
+ * some */
return EOF; /* Failed to recv data */
}
return (unsigned char) PqRecvBuffer[PqRecvPointer];
@@ -1249,49 +1270,12 @@ pq_getbyte_if_available(unsigned char *c)
Assert(PqCommReadingMsg);
- if (PqRecvPointer < PqRecvLength)
+ if (PqRecvPointer < PqRecvLength || (r = pq_recvbuf(true)) > 0)
{
*c = PqRecvBuffer[PqRecvPointer++];
return 1;
}
- /* Put the socket into non-blocking mode */
- socket_set_nonblocking(true);
-
- errno = 0;
-
- r = io_read_with_wait(MyProcPort, c, 1);
- if (r < 0)
- {
- /*
- * Ok if no data available without blocking or interrupted (though
- * EINTR really shouldn't happen with a non-blocking socket). Report
- * other errors.
- */
- if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)
- r = 0;
- else
- {
- /*
- * Careful: an ereport() that tries to write to the client would
- * cause recursion to here, leading to stack overflow and core
- * dump! This message must go *only* to the postmaster log.
- *
- * If errno is zero, assume it's EOF and let the caller complain.
- */
- if (errno != 0)
- ereport(COMMERROR,
- (errcode_for_socket_access(),
- errmsg("could not receive data from client: %m")));
- r = EOF;
- }
- }
- else if (r == 0)
- {
- /* EOF detected */
- r = EOF;
- }
-
return r;
}
@@ -1312,7 +1296,8 @@ pq_getbytes(char *s, size_t len)
{
while (PqRecvPointer >= PqRecvLength)
{
- if (pq_recvbuf()) /* If nothing in buffer, then recv some */
+ if (pq_recvbuf(false) == EOF) /* If nothing in buffer, then recv
+ * some */
return EOF; /* Failed to recv data */
}
amount = PqRecvLength - PqRecvPointer;
@@ -1346,7 +1331,8 @@ pq_discardbytes(size_t len)
{
while (PqRecvPointer >= PqRecvLength)
{
- if (pq_recvbuf()) /* If nothing in buffer, then recv some */
+ if (pq_recvbuf(false) == EOF) /* If nothing in buffer, then recv
+ * some */
return EOF; /* Failed to recv data */
}
amount = PqRecvLength - PqRecvPointer;
@@ -1574,7 +1560,7 @@ internal_flush(void)
char *bufptr = PqSendBuffer + PqSendStart;
char *bufend = PqSendBuffer + PqSendPointer;
- while (bufptr < bufend)
+ while (bufptr < bufend || io_stream_buffered_write_data(MyProcPort->io_stream) != 0)
{
int rc;
size_t bytes_sent;
@@ -1625,6 +1611,7 @@ internal_flush(void)
PqSendStart = PqSendPointer = 0;
ClientConnectionLost = 1;
InterruptPending = 1;
+ io_stream_reset_write_state(MyProcPort->io_stream);
return EOF;
}
@@ -1647,7 +1634,7 @@ socket_flush_if_writable(void)
int res;
/* Quick exit if nothing to do */
- if (PqSendPointer == PqSendStart)
+ if ((PqSendPointer == PqSendStart) && (io_stream_buffered_write_data(MyProcPort->io_stream) == 0))
return 0;
/* No-op if reentrant call */
@@ -1670,7 +1657,7 @@ socket_flush_if_writable(void)
static bool
socket_is_send_pending(void)
{
- return (PqSendStart < PqSendPointer);
+ return (PqSendStart < PqSendPointer || (io_stream_buffered_write_data(MyProcPort->io_stream) != 0));
}
/* --------------------------------
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index edf28e36f1..f06df197b7 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -99,6 +99,7 @@
#include "common/string.h"
#include "lib/ilist.h"
#include "libpq/auth.h"
+#include "libpq/compression.h"
#include "libpq/libpq.h"
#include "libpq/pqformat.h"
#include "libpq/pqsignal.h"
@@ -2210,12 +2211,15 @@ retry1:
valptr),
errhint("Valid values are: \"false\", 0, \"true\", 1, \"database\".")));
}
+ else if (strcmp(nameptr, "_pq_.libpq_compression") == 0)
+ {
+ configure_libpq_compression(port, valptr);
+ }
else if (strncmp(nameptr, "_pq_.", 5) == 0)
{
/*
* Any option beginning with _pq_. is reserved for use as a
- * protocol-level option, but at present no such options are
- * defined.
+ * protocol-level option.
*/
unrecognized_protocol_options =
lappend(unrecognized_protocol_options, pstrdup(nameptr));
@@ -4422,7 +4426,9 @@ BackendInitialize(Port *port)
* already did any appropriate error reporting.
*/
if (status != STATUS_OK)
+ {
proc_exit(0);
+ }
/*
* Now that we have the user and database name, we can set the process
diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
index 693b3669ba..52e8772ddb 100644
--- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
+++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
@@ -727,12 +727,15 @@ libpqrcv_PQgetResult(PGconn *streamConn)
* since we'll get interrupted by signals and can handle any
* interrupts here.
*/
- rc = WaitLatchOrSocket(MyLatch,
- WL_EXIT_ON_PM_DEATH | WL_SOCKET_READABLE |
- WL_LATCH_SET,
- PQsocket(streamConn),
- 0,
- WAIT_EVENT_LIBPQWALRECEIVER_RECEIVE);
+ if (PQreadPending(streamConn))
+ rc = WL_SOCKET_READABLE;
+ else
+ rc = WaitLatchOrSocket(MyLatch,
+ WL_EXIT_ON_PM_DEATH | WL_SOCKET_READABLE |
+ WL_LATCH_SET,
+ PQsocket(streamConn),
+ 0,
+ WAIT_EVENT_LIBPQWALRECEIVER_RECEIVE);
/* Interrupted? */
if (rc & WL_LATCH_SET)
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 3945a92ddd..a98f2ccd97 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -46,6 +46,7 @@
#include "common/scram-common.h"
#include "jit/jit.h"
#include "libpq/auth.h"
+#include "libpq/compression.h"
#include "libpq/libpq.h"
#include "libpq/scram.h"
#include "nodes/queryjumble.h"
@@ -4542,6 +4543,17 @@ struct config_string ConfigureNamesString[] =
NULL, NULL, NULL
},
+ {
+ {"libpq_compression", PGC_SIGHUP, CLIENT_CONN_OTHER,
+ gettext_noop("Sets the list of allowed libpq compression algorithms."),
+ NULL,
+ GUC_REPORT
+ },
+ &libpq_compress_algorithms,
+ "off",
+ check_libpq_compression, assign_libpq_compression, NULL
+ },
+
{
{"application_name", PGC_USERSET, LOGGING_WHAT,
gettext_noop("Sets the application name to be reported in statistics and logs."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index b2809c711a..0411571c02 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -74,6 +74,9 @@
# (change requires restart)
#bonjour_name = '' # defaults to the computer name
# (change requires restart)
+#libpq_compression = off # on to allow all supported compression
+ # methods; off to disable all compression;
+ # semicolon separated list of algorithms to allow some
# - TCP settings -
# see "man tcp" for details
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index 28d2ee435b..0c176e0940 100644
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -2649,7 +2649,7 @@ main(int argc, char **argv)
pg_fatal("unrecognized compression algorithm: \"%s\"",
compression_algorithm);
- parse_compress_specification(alg, compression_detail, &client_compress);
+ parse_compress_specification(alg, compression_detail, &client_compress, PG_COMPRESSION_OPTION_WORKERS | PG_COMPRESSION_OPTION_LONG_DISTANCE);
error_detail = validate_compress_specification(&client_compress);
if (error_detail != NULL)
pg_fatal("invalid compression specification: %s",
diff --git a/src/bin/pg_basebackup/pg_receivewal.c b/src/bin/pg_basebackup/pg_receivewal.c
index d0a4079d50..982035f1cd 100644
--- a/src/bin/pg_basebackup/pg_receivewal.c
+++ b/src/bin/pg_basebackup/pg_receivewal.c
@@ -803,7 +803,7 @@ main(int argc, char **argv)
compression_algorithm_str);
parse_compress_specification(compression_algorithm, compression_detail,
- &compression_spec);
+ &compression_spec, PG_COMPRESSION_OPTION_WORKERS | PG_COMPRESSION_OPTION_LONG_DISTANCE);
error_detail = validate_compress_specification(&compression_spec);
if (error_detail != NULL)
pg_fatal("invalid compression specification: %s",
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 050a831226..81d515bd26 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -759,7 +759,7 @@ main(int argc, char **argv)
compression_algorithm_str);
parse_compress_specification(compression_algorithm, compression_detail,
- &compression_spec);
+ &compression_spec, PG_COMPRESSION_OPTION_LONG_DISTANCE);
error_detail = validate_compress_specification(&compression_spec);
if (error_detail != NULL)
pg_fatal("invalid compression specification: %s",
@@ -769,15 +769,6 @@ main(int argc, char **argv)
if (error_detail != NULL)
pg_fatal("%s", error_detail);
- /*
- * Disable support for zstd workers for now - these are based on
- * threading, and it's unclear how it interacts with parallel dumps on
- * platforms where that relies on threads too (e.g. Windows).
- */
- if (compression_spec.options & PG_COMPRESSION_OPTION_WORKERS)
- pg_log_warning("compression option \"%s\" is not currently supported by pg_dump",
- "workers");
-
/*
* If emitting an archive format, we always want to emit a DATABASE item,
* in case --create is specified at pg_restore time.
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 1253f1989e..8754d503b4 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -7433,6 +7433,9 @@ threadRun(void *arg)
int nsocks; /* number of sockets to be waited for */
pg_time_usec_t min_usec;
pg_time_usec_t now = 0; /* set this only if needed */
+ bool buffered_rx = false; /* true if some of the clients has
+ * data left in SSL/ZPQ read
+ * buffers */
/*
* identify which client sockets should be checked for input, and
@@ -7468,6 +7471,9 @@ threadRun(void *arg)
*/
int sock = PQsocket(st->con);
+ /* check if conn has buffered SSL / ZPQ read data */
+ buffered_rx = buffered_rx || PQreadPending(st->con);
+
if (sock < 0)
{
pg_log_error("invalid socket: %s", PQerrorMessage(st->con));
@@ -7511,7 +7517,7 @@ threadRun(void *arg)
{
if (nsocks > 0)
{
- rc = wait_on_socket_set(sockets, min_usec);
+ rc = buffered_rx ? 1 : wait_on_socket_set(sockets, min_usec);
}
else /* nothing active, simple sleep */
{
@@ -7520,7 +7526,7 @@ threadRun(void *arg)
}
else /* no explicit delay, wait without timeout */
{
- rc = wait_on_socket_set(sockets, 0);
+ rc = buffered_rx ? 1 : wait_on_socket_set(sockets, 0);
}
if (rc < 0)
@@ -7560,8 +7566,11 @@ threadRun(void *arg)
pg_log_error("invalid socket: %s", PQerrorMessage(st->con));
goto done;
}
-
- if (!socket_has_input(sockets, sock, nsocks++))
+ if (PQreadPending(st->con))
+ {
+ nsocks++;
+ }
+ else if (!socket_has_input(sockets, sock, nsocks++))
continue;
}
else if (st->state == CSTATE_FINISHED ||
diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c
index 82cc091568..4ed6fb5d21 100644
--- a/src/bin/psql/command.c
+++ b/src/bin/psql/command.c
@@ -172,6 +172,7 @@ static int count_lines_in_buf(PQExpBuffer buf);
static void print_with_linenumbers(FILE *output, char *lines, bool is_func);
static void minimal_error_message(PGresult *res);
+static void printCompressionInfo(void);
static void printSSLInfo(void);
static void printGSSInfo(void);
static bool printPsetInfo(const char *param, printQueryOpt *popt);
@@ -676,6 +677,7 @@ exec_command_conninfo(PsqlScanState scan_state, bool active_branch)
printf(_("You are connected to database \"%s\" as user \"%s\" on host \"%s\" at port \"%s\".\n"),
db, PQuser(pset.db), host, PQport(pset.db));
}
+ printCompressionInfo();
printSSLInfo();
printGSSInfo();
}
@@ -3821,6 +3823,22 @@ connection_warnings(bool in_startup)
}
}
+/*
+ * printCompressionInfo
+ *
+ * Print information about used compressor/decompressor
+ */
+static void
+printCompressionInfo(void)
+{
+ char *algorithms = PQcompression(pset.db);
+
+ if (algorithms != NULL)
+ {
+ printf(_("Compression: server: %s, client: %s\n"), PQserverCompression(pset.db), algorithms);
+ pfree(algorithms);
+ }
+}
/*
* printSSLInfo
diff --git a/src/common/Makefile b/src/common/Makefile
index a80f4b39a6..49dc539e18 100644
--- a/src/common/Makefile
+++ b/src/common/Makefile
@@ -83,7 +83,9 @@ OBJS_COMMON = \
unicode_norm.o \
username.o \
wait_error.o \
- wchar.o
+ wchar.o \
+ z_stream.o \
+ zpq_stream.o
ifeq ($(with_ssl),openssl)
OBJS_COMMON += \
diff --git a/src/common/compression.c b/src/common/compression.c
index ee937623f0..7faf788f01 100644
--- a/src/common/compression.c
+++ b/src/common/compression.c
@@ -36,6 +36,16 @@
#include "common/compression.h"
+#ifndef FRONTEND
+#define ALLOC palloc
+#define STRDUP pstrdup
+#define FREE pfree
+#else
+#define ALLOC malloc
+#define STRDUP strdup
+#define FREE free
+#endif
+
static int expect_integer_value(char *keyword, char *value,
pg_compress_specification *result);
static bool expect_boolean_value(char *keyword, char *value,
@@ -105,7 +115,7 @@ get_compress_algorithm_name(pg_compress_algorithm algorithm)
*/
void
parse_compress_specification(pg_compress_algorithm algorithm, char *specification,
- pg_compress_specification *result)
+ pg_compress_specification *result, unsigned allowed_options)
{
int bare_level;
char *bare_level_endp;
@@ -113,7 +123,14 @@ parse_compress_specification(pg_compress_algorithm algorithm, char *specificatio
/* Initial setup of result object. */
result->algorithm = algorithm;
result->options = 0;
- result->parse_error = NULL;
+ result->has_error = false;
+
+ /*
+ * By default we allow both compression and decompression unless
+ * explicitly disabled
+ */
+ result->compress = true;
+ result->decompress = true;
/*
* Assign a default level depending on the compression method. This may
@@ -128,27 +145,27 @@ parse_compress_specification(pg_compress_algorithm algorithm, char *specificatio
#ifdef USE_LZ4
result->level = 0; /* fast compression mode */
#else
- result->parse_error =
- psprintf(_("this build does not support compression with %s"),
- "LZ4");
+ result->has_error = true;
+ snprintf(result->parse_error, 255, _("this build does not support compression with %s"),
+ "LZ4");
#endif
break;
case PG_COMPRESSION_ZSTD:
#ifdef USE_ZSTD
result->level = ZSTD_CLEVEL_DEFAULT;
#else
- result->parse_error =
- psprintf(_("this build does not support compression with %s"),
- "ZSTD");
+ result->has_error = true;
+ snprintf(result->parse_error, 255, _("this build does not support compression with %s"),
+ "ZSTD");
#endif
break;
case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
result->level = Z_DEFAULT_COMPRESSION;
#else
- result->parse_error =
- psprintf(_("this build does not support compression with %s"),
- "gzip");
+ result->has_error = true;
+ snprintf(result->parse_error, 255, _("this build does not support compression with %s"),
+ "gzip");
#endif
break;
}
@@ -201,20 +218,20 @@ parse_compress_specification(pg_compress_algorithm algorithm, char *specificatio
/* Reject empty keyword. */
if (kwlen == 0)
{
- result->parse_error =
- pstrdup(_("found empty string where a compression option was expected"));
+ result->has_error = true;
+ strlcpy(result->parse_error, _("found empty string where a compression option was expected"), 255);
break;
}
/* Extract keyword and value as separate C strings. */
- keyword = palloc(kwlen + 1);
+ keyword = ALLOC(kwlen + 1);
memcpy(keyword, kwstart, kwlen);
keyword[kwlen] = '\0';
if (!has_value)
value = NULL;
else
{
- value = palloc(vlen + 1);
+ value = ALLOC(vlen + 1);
memcpy(value, vstart, vlen);
value[vlen] = '\0';
}
@@ -229,24 +246,36 @@ parse_compress_specification(pg_compress_algorithm algorithm, char *specificatio
* set at least thanks to the logic above.
*/
}
- else if (strcmp(keyword, "workers") == 0)
+ else if (strcmp(keyword, "workers") == 0 && (allowed_options & PG_COMPRESSION_OPTION_WORKERS))
{
result->workers = expect_integer_value(keyword, value, result);
result->options |= PG_COMPRESSION_OPTION_WORKERS;
}
- else if (strcmp(keyword, "long") == 0)
+ else if (strcmp(keyword, "long") == 0 && (allowed_options & PG_COMPRESSION_OPTION_LONG_DISTANCE))
{
result->long_distance = expect_boolean_value(keyword, value, result);
result->options |= PG_COMPRESSION_OPTION_LONG_DISTANCE;
}
+ else if (strcmp(keyword, "compress") == 0 && (allowed_options & PG_COMPRESSION_OPTION_COMPRESS))
+ {
+ result->compress = expect_boolean_value(keyword, value, result);
+ result->options |= PG_COMPRESSION_OPTION_COMPRESS;
+ }
+ else if (strcmp(keyword, "decompress") == 0 && (allowed_options & PG_COMPRESSION_OPTION_DECOMPRESS))
+ {
+ result->compress = expect_boolean_value(keyword, value, result);
+ result->options |= PG_COMPRESSION_OPTION_DECOMPRESS;
+ }
else
- result->parse_error =
- psprintf(_("unrecognized compression option: \"%s\""), keyword);
+ {
+ result->has_error = true;
+ snprintf(result->parse_error, 255, _("unrecognized compression option: \"%s\" %d"), keyword, allowed_options);
+ }
/* Release memory, just to be tidy. */
- pfree(keyword);
+ FREE(keyword);
if (value != NULL)
- pfree(value);
+ FREE(value);
/*
* If we got an error or have reached the end of the string, stop.
@@ -256,7 +285,7 @@ parse_compress_specification(pg_compress_algorithm algorithm, char *specificatio
* keyword cannot have been the end of the string, but the end of the
* value might have been.
*/
- if (result->parse_error != NULL ||
+ if (result->has_error ||
(vend == NULL ? *kwend == '\0' : *vend == '\0'))
break;
@@ -265,10 +294,71 @@ parse_compress_specification(pg_compress_algorithm algorithm, char *specificatio
}
}
+/*
+ * Serialize the compression specification as parse_compress_specification would parse it.
+ *
+ * Processes arguments like snprintf, where the returned value is the length of the serialized
+ * string even if buffer is NULL and buffer_size is 0 (useful for allocating a buffer to hold
+ * serialized values)
+ */
+int
+serialize_compress_specification(const pg_compress_specification *spec, char *buffer, size_t buffer_size)
+{
+ int length = 0;
+ int n;
+
+ n = snprintf(buffer, buffer_size, "level=%d", spec->level);
+ length += n;
+ if (buffer)
+ {
+ buffer += n;
+ }
+
+ /* Write the options, if any. */
+ if (spec->options & PG_COMPRESSION_OPTION_WORKERS)
+ {
+ n = snprintf(buffer, buffer_size > 0 ? buffer_size - length : 0, ",workers=%d", spec->workers);
+ length += n;
+ if (buffer)
+ {
+ buffer += n;
+ }
+ }
+ if (spec->options & PG_COMPRESSION_OPTION_LONG_DISTANCE)
+ {
+ n = snprintf(buffer, buffer_size > 0 ? buffer_size - length : 0, ",long=%s", spec->long_distance ? "yes" : "no");
+ length += n;
+ if (buffer)
+ {
+ buffer += n;
+ }
+ }
+ if (spec->options & PG_COMPRESSION_OPTION_COMPRESS)
+ {
+ n = snprintf(buffer, buffer_size > 0 ? buffer_size - length : 0, ",compress=%s", spec->compress ? "yes" : "no");
+ length += n;
+ if (buffer)
+ {
+ buffer += n;
+ }
+ }
+ if (spec->options & PG_COMPRESSION_OPTION_DECOMPRESS)
+ {
+ n = snprintf(buffer, buffer_size > 0 ? buffer_size - length : 0, ",decompress=%s", spec->decompress ? "yes" : "no");
+ length += n;
+ if (buffer)
+ {
+ buffer += n;
+ }
+ }
+
+ return length;
+}
+
/*
* Parse 'value' as an integer and return the result.
*
- * If parsing fails, set result->parse_error to an appropriate message
+ * If parsing fails, set result->has_error and write an appropriate message to result->parse_error
* and return -1.
*/
static int
@@ -279,18 +369,17 @@ expect_integer_value(char *keyword, char *value, pg_compress_specification *resu
if (value == NULL)
{
- result->parse_error =
- psprintf(_("compression option \"%s\" requires a value"),
- keyword);
+ result->has_error = true;
+ snprintf(result->parse_error, 255, _("compression option \"%s\" requires a value"), keyword);
return -1;
}
ivalue = strtol(value, &ivalue_endp, 10);
if (ivalue_endp == value || *ivalue_endp != '\0')
{
- result->parse_error =
- psprintf(_("value for compression option \"%s\" must be an integer"),
- keyword);
+ result->has_error = true;
+ snprintf(result->parse_error, 255, _("value for compression option \"%s\" must be an integer"),
+ keyword);
return -1;
}
return ivalue;
@@ -299,8 +388,8 @@ expect_integer_value(char *keyword, char *value, pg_compress_specification *resu
/*
* Parse 'value' as a boolean and return the result.
*
- * If parsing fails, set result->parse_error to an appropriate message
- * and return -1. The caller must check result->parse_error to determine if
+ * If parsing fails, set result->has_error and write an appropriate message to result->parse_error
+ * and return -1. The caller must check result->has_error to determine if
* the call was successful.
*
* Valid values are: yes, no, on, off, 1, 0.
@@ -327,9 +416,10 @@ expect_boolean_value(char *keyword, char *value, pg_compress_specification *resu
if (pg_strcasecmp(value, "0") == 0)
return false;
- result->parse_error =
- psprintf(_("value for compression option \"%s\" must be a Boolean value"),
- keyword);
+
+ result->has_error = true;
+ snprintf(result->parse_error, 255, _("value for compression option \"%s\" must be a Boolean value"),
+ keyword);
return false;
}
@@ -348,7 +438,7 @@ validate_compress_specification(pg_compress_specification *spec)
int default_level = 0;
/* If it didn't even parse OK, it's definitely no good. */
- if (spec->parse_error != NULL)
+ if (spec->has_error)
return spec->parse_error;
/*
@@ -376,16 +466,22 @@ validate_compress_specification(pg_compress_specification *spec)
break;
case PG_COMPRESSION_NONE:
if (spec->level != 0)
- return psprintf(_("compression algorithm \"%s\" does not accept a compression level"),
- get_compress_algorithm_name(spec->algorithm));
+ {
+ snprintf(spec->parse_error, 255, _("compression algorithm \"%s\" does not accept a compression level"),
+ get_compress_algorithm_name(spec->algorithm));
+ return spec->parse_error;
+ }
break;
}
if ((spec->level < min_level || spec->level > max_level) &&
spec->level != default_level)
- return psprintf(_("compression algorithm \"%s\" expects a compression level between %d and %d (default at %d)"),
- get_compress_algorithm_name(spec->algorithm),
- min_level, max_level, default_level);
+ {
+ snprintf(spec->parse_error, 255, _("compression algorithm \"%s\" expects a compression level between %d and %d (default at %d)"),
+ get_compress_algorithm_name(spec->algorithm),
+ min_level, max_level, default_level);
+ return spec->parse_error;
+ }
/*
* Of the compression algorithms that we currently support, only zstd
@@ -394,8 +490,9 @@ validate_compress_specification(pg_compress_specification *spec)
if ((spec->options & PG_COMPRESSION_OPTION_WORKERS) != 0 &&
(spec->algorithm != PG_COMPRESSION_ZSTD))
{
- return psprintf(_("compression algorithm \"%s\" does not accept a worker count"),
- get_compress_algorithm_name(spec->algorithm));
+ snprintf(spec->parse_error, 255, _("compression algorithm \"%s\" does not accept a worker count"),
+ get_compress_algorithm_name(spec->algorithm));
+ return spec->parse_error;
}
/*
@@ -405,13 +502,45 @@ validate_compress_specification(pg_compress_specification *spec)
if ((spec->options & PG_COMPRESSION_OPTION_LONG_DISTANCE) != 0 &&
(spec->algorithm != PG_COMPRESSION_ZSTD))
{
- return psprintf(_("compression algorithm \"%s\" does not support long-distance mode"),
- get_compress_algorithm_name(spec->algorithm));
+ snprintf(spec->parse_error, 255, _("compression algorithm \"%s\" does not support long-distance mode"),
+ get_compress_algorithm_name(spec->algorithm));
+ return spec->parse_error;
}
return NULL;
}
+bool
+supported_compression_algorithm(pg_compress_algorithm algorithm)
+{
+ switch (algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ return true;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ return true;
+#else
+ return false;
+#endif
+ case PG_COMPRESSION_LZ4:
+#ifdef USE_LZ4
+ return true;
+#else
+ return false;
+#endif
+ case PG_COMPRESSION_ZSTD:
+#ifdef USE_ZSTD
+ return true;
+#else
+ return false;
+#endif
+ /* no default, to provoke compiler warnings if values are added */
+ }
+ Assert(false);
+ return false; /* placate compiler */
+}
+
#ifdef FRONTEND
/*
@@ -440,13 +569,13 @@ parse_compress_options(const char *option, char **algorithm, char **detail)
{
if (result == 0)
{
- *algorithm = pstrdup("none");
+ *algorithm = STRDUP("none");
*detail = NULL;
}
else
{
- *algorithm = pstrdup("gzip");
- *detail = pstrdup(option);
+ *algorithm = STRDUP("gzip");
+ *detail = STRDUP(option);
}
return;
}
@@ -458,19 +587,19 @@ parse_compress_options(const char *option, char **algorithm, char **detail)
sep = strchr(option, ':');
if (sep == NULL)
{
- *algorithm = pstrdup(option);
+ *algorithm = STRDUP(option);
*detail = NULL;
}
else
{
char *alg;
- alg = palloc((sep - option) + 1);
+ alg = ALLOC((sep - option) + 1);
memcpy(alg, option, sep - option);
alg[sep - option] = '\0';
*algorithm = alg;
- *detail = pstrdup(sep + 1);
+ *detail = STRDUP(sep + 1);
}
}
#endif /* FRONTEND */
diff --git a/src/common/io_stream.c b/src/common/io_stream.c
index b15aca326d..6598d12841 100644
--- a/src/common/io_stream.c
+++ b/src/common/io_stream.c
@@ -129,6 +129,19 @@ io_stream_buffered_write_data(IoStream * stream)
return false;
}
+void
+io_stream_reset_write_state(IoStream * stream)
+{
+
+ IoStreamLayer *layer;
+
+ for (layer = stream->layer; layer != NULL; layer = layer->next)
+ {
+ if (layer->processor->reset_write_state != NULL)
+ layer->processor->reset_write_state(layer->context);
+ }
+}
+
ssize_t
io_stream_next_read(IoStreamLayer * layer, void *data, size_t size, bool buffered_only)
{
diff --git a/src/common/meson.build b/src/common/meson.build
index e25fa44133..99ef06cd4c 100644
--- a/src/common/meson.build
+++ b/src/common/meson.build
@@ -38,6 +38,8 @@ common_sources = files(
'username.c',
'wait_error.c',
'wchar.c',
+ 'z_stream.c',
+ 'zpq_stream.c'
)
if ssl.found()
diff --git a/src/common/z_stream.c b/src/common/z_stream.c
new file mode 100644
index 0000000000..eb34733702
--- /dev/null
+++ b/src/common/z_stream.c
@@ -0,0 +1,995 @@
+/*-------------------------------------------------------------------------
+ *
+ * z_stream.c
+ * Functions implementing streaming compression algorithms
+ *
+ * Copyright (c) 2018-2023, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/common/z_stream.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "c.h"
+#include "pg_config.h"
+#include "common/z_stream.h"
+#include "utils/elog.h"
+
+typedef struct
+{
+ /*
+ * Id of compression algorithm.
+ */
+ pg_compress_algorithm algorithm;
+
+ /*
+ * Create new compression stream. level: compression level
+ */
+ void *(*create_compressor) (int level);
+
+ /*
+ * Create new decompression stream.
+ */
+ void *(*create_decompressor) ();
+
+ /*
+ * Decompress up to "src_size" compressed bytes from *src and write up to
+ * "dst_size" raw (decompressed) bytes to *dst. Number of decompressed
+ * bytes written to *dst is stored in *dst_processed. Number of compressed
+ * bytes read from *src is stored in *src_processed.
+ *
+ * Return codes: ZS_OK if no errors were encountered during decompression
+ * attempt. This return code does not guarantee that *src_processed > 0 or
+ * *dst_processed > 0.
+ *
+ * ZS_STREAM_END if encountered end of compressed data stream.
+ *
+ * ZS_DECOMPRESS_ERROR if encountered an error during decompression
+ * attempt.
+ */
+ int (*decompress) (void *ds, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed);
+
+ /*
+ * Returns true if there is some data left in internal decompression
+ * buffers
+ */
+ bool (*decompress_buffered_data) (void *ds);
+
+ /*
+ * Compress up to "src_size" raw (non-compressed) bytes from *src and
+ * write up to "dst_size" compressed bytes to *dst. Number of compressed
+ * bytes written to *dst is stored in *dst_processed. Number of
+ * non-compressed bytes read from *src is stored in *src_processed.
+ *
+ * Return codes: ZS_OK if no errors were encountered during compression
+ * attempt. This return code does not guarantee that *src_processed > 0 or
+ * *dst_processed > 0.
+ *
+ * ZS_COMPRESS_ERROR if encountered an error during compression attempt.
+ */
+ int (*compress) (void *cs, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed);
+
+ /*
+ * Returns true if there is some data left in internal compression buffers
+ */
+ bool (*compress_buffered_data) (void *ds);
+
+ /*
+ * Free compression stream created by create_compressor function.
+ */
+ void (*free_compressor) (void *cs);
+
+ /*
+ * Free decompression stream created by create_decompressor function.
+ */
+ void (*free_decompressor) (void *ds);
+
+ /*
+ * Get compressor error message.
+ */
+ char const *(*compress_error) (void *cs);
+
+ /*
+ * Get decompressor error message.
+ */
+ char const *(*decompress_error) (void *ds);
+
+ int (*end_compression) (void *cs, void *dst, size_t dst_size, size_t *dst_processed);
+} ZAlgorithm;
+
+struct ZStream
+{
+ ZAlgorithm const *algorithm;
+ void *stream;
+};
+
+#ifndef FRONTEND
+#include "utils/palloc.h"
+#include "utils/memutils.h"
+#define ALLOC(size) MemoryContextAlloc(TopMemoryContext, size)
+#define FREE(size) pfree(size)
+#else
+#define ALLOC(size) malloc(size)
+#define FREE(size) free(size)
+#endif
+
+#if HAVE_LIBZSTD
+
+#include <stdlib.h>
+#include <zstd.h>
+
+/*
+ * Maximum allowed back-reference distance, expressed as power of 2.
+ * This setting controls max compressor/decompressor window size.
+ * More details https://github.com/facebook/zstd/blob/v1.4.7/lib/zstd.h#L536
+ */
+#define ZSTD_WINDOWLOG_LIMIT 23 /* set max window size to 8MB */
+
+
+typedef struct ZS_ZSTD_CStream
+{
+ ZSTD_CStream *stream;
+ bool has_buffered_data;
+ char const *error; /* error message */
+} ZS_ZSTD_CStream;
+
+typedef struct ZS_ZSTD_DStream
+{
+ ZSTD_DStream *stream;
+ bool has_buffered_data;
+ char const *error; /* error message */
+} ZS_ZSTD_DStream;
+
+static void *
+zstd_create_compressor(int level)
+{
+ size_t rc;
+ ZS_ZSTD_CStream *c_stream = (ZS_ZSTD_CStream *) ALLOC(sizeof(ZS_ZSTD_CStream));
+
+ c_stream->stream = ZSTD_createCStream();
+ c_stream->has_buffered_data = false;
+ rc = ZSTD_initCStream(c_stream->stream, level);
+ if (ZSTD_isError(rc))
+ {
+ ZSTD_freeCStream(c_stream->stream);
+ FREE(c_stream);
+ return NULL;
+ }
+#if ZSTD_VERSION_MAJOR > 1 || ZSTD_VERSION_MINOR > 3
+ ZSTD_CCtx_setParameter(c_stream->stream, ZSTD_c_windowLog, ZSTD_WINDOWLOG_LIMIT);
+#endif
+ c_stream->error = NULL;
+ return c_stream;
+}
+
+static void *
+zstd_create_decompressor()
+{
+ size_t rc;
+ ZS_ZSTD_DStream *d_stream = (ZS_ZSTD_DStream *) ALLOC(sizeof(ZS_ZSTD_DStream));
+
+ d_stream->stream = ZSTD_createDStream();
+ d_stream->has_buffered_data = false;
+ rc = ZSTD_initDStream(d_stream->stream);
+ if (ZSTD_isError(rc))
+ {
+ ZSTD_freeDStream(d_stream->stream);
+ FREE(d_stream);
+ return NULL;
+ }
+#if ZSTD_VERSION_MAJOR > 1 || ZSTD_VERSION_MINOR > 3
+ ZSTD_DCtx_setParameter(d_stream->stream, ZSTD_d_windowLogMax, ZSTD_WINDOWLOG_LIMIT);
+#endif
+ d_stream->error = NULL;
+ return d_stream;
+}
+
+static int
+zstd_decompress(void *d_stream, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed)
+{
+ ZS_ZSTD_DStream *ds = (ZS_ZSTD_DStream *) d_stream;
+ ZSTD_inBuffer in;
+ ZSTD_outBuffer out;
+ size_t rc;
+
+ in.src = src;
+ in.pos = 0;
+ in.size = src_size;
+
+ out.dst = dst;
+ out.pos = 0;
+ out.size = dst_size;
+
+ rc = ZSTD_decompressStream(ds->stream, &out, &in);
+
+ *src_processed = in.pos;
+ *dst_processed = out.pos;
+ if (ZSTD_isError(rc))
+ {
+ ds->error = ZSTD_getErrorName(rc);
+ return ZS_DECOMPRESS_ERROR;
+ }
+
+ if (rc == 0)
+ {
+ return ZS_STREAM_END;
+ }
+
+ /*
+ * if `output.pos == output.size`, there might be some data left within
+ * internal buffers
+ */
+ ds->has_buffered_data = out.pos == out.size;
+
+ return ZS_OK;
+}
+
+static bool
+zstd_decompress_buffered_data(void *d_stream)
+{
+ ZS_ZSTD_DStream *ds = (ZS_ZSTD_DStream *) d_stream;
+
+ return ds->has_buffered_data;
+}
+
+static int
+zstd_compress(void *c_stream, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed)
+{
+ ZS_ZSTD_CStream *cs = (ZS_ZSTD_CStream *) c_stream;
+ ZSTD_inBuffer in;
+ ZSTD_outBuffer out;
+
+ in.src = src;
+ in.pos = 0;
+ in.size = src_size;
+
+ out.dst = dst;
+ out.pos = 0;
+ out.size = dst_size;
+
+ if (in.pos < src_size) /* Has something to compress in input buffer */
+ {
+ size_t rc = ZSTD_compressStream(cs->stream, &out, &in);
+
+ *dst_processed = out.pos;
+ *src_processed = in.pos;
+ if (ZSTD_isError(rc))
+ {
+ cs->error = ZSTD_getErrorName(rc);
+ return ZS_COMPRESS_ERROR;
+ }
+ }
+
+ if (in.pos == src_size) /* All data is compressed: flush internal zstd
+ * buffer */
+ {
+ size_t tx_not_flushed = ZSTD_flushStream(cs->stream, &out);
+
+ *dst_processed = out.pos;
+ cs->has_buffered_data = tx_not_flushed > 0;
+ }
+ else
+ {
+ cs->has_buffered_data = false;
+ }
+
+ return ZS_OK;
+}
+
+static bool
+zstd_compress_buffered_data(void *c_stream)
+{
+ ZS_ZSTD_CStream *cs = (ZS_ZSTD_CStream *) c_stream;
+
+ return cs->has_buffered_data;
+}
+
+static int
+zstd_end(void *c_stream, void *dst, size_t dst_size, size_t *dst_processed)
+{
+ size_t tx_not_flushed;
+ ZS_ZSTD_CStream *cs = (ZS_ZSTD_CStream *) c_stream;
+ ZSTD_outBuffer output;
+
+ output.dst = dst;
+ output.pos = 0;
+ output.size = dst_size;
+
+ do
+ {
+ tx_not_flushed = ZSTD_endStream(cs->stream, &output);
+ } while ((tx_not_flushed > 0) && (output.pos < output.size));
+
+ *dst_processed = output.pos;
+
+ cs->has_buffered_data = tx_not_flushed > 0;
+ return ZS_OK;
+}
+
+static void
+zstd_free_compressor(void *c_stream)
+{
+ ZS_ZSTD_CStream *cs = (ZS_ZSTD_CStream *) c_stream;
+
+ if (cs != NULL)
+ {
+ ZSTD_freeCStream(cs->stream);
+ FREE(cs);
+ }
+}
+
+static void
+zstd_free_decompressor(void *d_stream)
+{
+ ZS_ZSTD_DStream *ds = (ZS_ZSTD_DStream *) d_stream;
+
+ if (ds != NULL)
+ {
+ ZSTD_freeDStream(ds->stream);
+ FREE(ds);
+ }
+}
+
+static char const *
+zstd_compress_error(void *c_stream)
+{
+ ZS_ZSTD_CStream *cs = (ZS_ZSTD_CStream *) c_stream;
+
+ return cs->error;
+}
+
+static char const *
+zstd_decompress_error(void *d_stream)
+{
+ ZS_ZSTD_DStream *ds = (ZS_ZSTD_DStream *) d_stream;
+
+ return ds->error;
+}
+
+static ZAlgorithm const zstd_algorithm = {
+ .algorithm = PG_COMPRESSION_ZSTD,
+ .create_compressor = zstd_create_compressor,
+ .create_decompressor = zstd_create_decompressor,
+ .decompress = zstd_decompress,
+ .decompress_buffered_data = zstd_decompress_buffered_data,
+ .compress = zstd_compress,
+ .compress_buffered_data = zstd_compress_buffered_data,
+ .free_compressor = zstd_free_compressor,
+ .free_decompressor = zstd_free_decompressor,
+ .compress_error = zstd_compress_error,
+ .decompress_error = zstd_decompress_error,
+ .end_compression = zstd_end
+};
+
+#endif
+
+#if HAVE_LIBZ
+
+#include <stdlib.h>
+#include <zlib.h>
+
+
+static void *
+zlib_create_compressor(int level)
+{
+ int rc;
+ z_stream *c_stream = (z_stream *) ALLOC(sizeof(z_stream));
+
+ memset(c_stream, 0, sizeof(*c_stream));
+ rc = deflateInit(c_stream, level);
+ if (rc != Z_OK)
+ {
+ FREE(c_stream);
+ return NULL;
+ }
+ return c_stream;
+}
+
+static void *
+zlib_create_decompressor()
+{
+ int rc;
+ z_stream *d_stream = (z_stream *) ALLOC(sizeof(z_stream));
+
+ memset(d_stream, 0, sizeof(*d_stream));
+ rc = inflateInit(d_stream);
+ if (rc != Z_OK)
+ {
+ FREE(d_stream);
+ return NULL;
+ }
+ return d_stream;
+}
+
+static int
+zlib_decompress(void *d_stream, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed)
+{
+ z_stream *ds = (z_stream *) d_stream;
+ int rc;
+
+ ds->next_in = (Bytef *) src;
+ ds->avail_in = src_size;
+ ds->next_out = (Bytef *) dst;
+ ds->avail_out = dst_size;
+
+ rc = inflate(ds, Z_SYNC_FLUSH);
+ *src_processed = src_size - ds->avail_in;
+ *dst_processed = dst_size - ds->avail_out;
+
+ if (rc == Z_STREAM_END)
+ {
+ return ZS_STREAM_END;
+ }
+ if (rc != Z_OK && rc != Z_BUF_ERROR)
+ {
+ return ZS_DECOMPRESS_ERROR;
+ }
+
+ return ZS_OK;
+}
+
+static bool
+zlib_decompress_buffered_data(void *d_stream)
+{
+ z_stream *ds = (z_stream *) d_stream;
+ unsigned deflate_pending = 0;
+
+ return deflatePending(ds, &deflate_pending, Z_NULL) > 0;
+}
+
+static int
+zlib_compress(void *c_stream, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed)
+{
+ z_stream *cs = (z_stream *) c_stream;
+ int rc PG_USED_FOR_ASSERTS_ONLY;
+
+ cs->next_out = (Bytef *) dst;
+ cs->avail_out = dst_size;
+ cs->next_in = (Bytef *) src;
+ cs->avail_in = src_size;
+
+ rc = deflate(cs, Z_SYNC_FLUSH);
+ Assert(rc == Z_OK);
+ *dst_processed = dst_size - cs->avail_out;
+ *src_processed = src_size - cs->avail_in;
+
+ return ZS_OK;
+}
+
+static bool
+zlib_compress_buffered_data(void *c_stream)
+{
+ return false;
+}
+
+static int
+zlib_end(void *c_stream, void *dst, size_t dst_size, size_t *dst_processed)
+{
+ z_stream *cs = (z_stream *) c_stream;
+ int rc;
+
+ cs->next_out = (Bytef *) dst;
+ cs->avail_out = dst_size;
+ cs->next_in = NULL;
+ cs->avail_in = 0;
+
+ rc = deflate(cs, Z_STREAM_END);
+ Assert(rc == Z_OK || rc == Z_STREAM_END);
+ *dst_processed = dst_size - cs->avail_out;
+ if (rc == Z_STREAM_END)
+ {
+ return ZS_OK;
+ }
+
+ return rc;
+}
+
+static void
+zlib_free_compressor(void *c_stream)
+{
+ z_stream *cs = (z_stream *) c_stream;
+
+ if (cs != NULL)
+ {
+ deflateEnd(cs);
+ FREE(cs);
+ }
+}
+
+static void
+zlib_free_decompressor(void *d_stream)
+{
+ z_stream *ds = (z_stream *) d_stream;
+
+ if (ds != NULL)
+ {
+ inflateEnd(ds);
+ FREE(ds);
+ }
+}
+
+static char const *
+zlib_error(void *stream)
+{
+ z_stream *zs = (z_stream *) stream;
+
+ return zs->msg;
+}
+
+/* as with elsewhere in postgres, gzip really means zlib */
+static ZAlgorithm const zlib_algorithm = {
+ .algorithm = PG_COMPRESSION_GZIP,
+ .create_compressor = zlib_create_compressor,
+ .create_decompressor = zlib_create_decompressor,
+ .decompress = zlib_decompress,
+ .decompress_buffered_data = zlib_decompress_buffered_data,
+ .compress = zlib_compress,
+ .compress_buffered_data = zlib_compress_buffered_data,
+ .free_compressor = zlib_free_compressor,
+ .free_decompressor = zlib_free_decompressor,
+ .compress_error = zlib_error,
+ .decompress_error = zlib_error,
+ .end_compression = zlib_end
+};
+
+#endif
+
+#if USE_LZ4
+#include <lz4.h>
+
+#define MESSAGE_MAX_BYTES 64 * 1024
+#define RING_BUFFER_BYTES (LZ4_DECODER_RING_BUFFER_SIZE(MESSAGE_MAX_BYTES))
+
+typedef struct ZS_LZ4_CStream
+{
+ LZ4_stream_t *stream;
+ int level;
+ size_t buf_pos;
+ char *last_error;
+ char buf[RING_BUFFER_BYTES];
+} ZS_LZ4_CStream;
+
+typedef struct ZS_LZ4_DStream
+{
+ LZ4_streamDecode_t *stream;
+ size_t buf_pos;
+ size_t read_pos;
+ char *last_error;
+ char buf[RING_BUFFER_BYTES];
+} ZS_LZ4_DStream;
+
+static void *
+lz4_create_compressor(int level)
+{
+ ZS_LZ4_CStream *c_stream = (ZS_LZ4_CStream *) ALLOC(sizeof(ZS_LZ4_CStream));
+
+ if (c_stream == NULL)
+ {
+ return NULL;
+ }
+ c_stream->stream = LZ4_createStream();
+ c_stream->level = level;
+ c_stream->buf_pos = 0;
+ if (c_stream->stream == NULL)
+ {
+ FREE(c_stream);
+ return NULL;
+ }
+ return c_stream;
+}
+
+static void *
+lz4_create_decompressor()
+{
+ ZS_LZ4_DStream *d_stream = (ZS_LZ4_DStream *) ALLOC(sizeof(ZS_LZ4_DStream));
+
+ if (d_stream == NULL)
+ {
+ return NULL;
+ }
+
+ d_stream->stream = LZ4_createStreamDecode();
+ d_stream->buf_pos = 0;
+ d_stream->read_pos = 0;
+ if (d_stream->stream == NULL)
+ {
+ FREE(d_stream);
+ return NULL;
+ }
+
+ return d_stream;
+}
+char last_error[256];
+
+static int
+lz4_decompress(void *d_stream, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed)
+{
+ ZS_LZ4_DStream *ds = (ZS_LZ4_DStream *) d_stream;
+ size_t copyable;
+ char *decPtr;
+ int decBytes;
+
+ if (ds->read_pos < ds->buf_pos)
+ {
+ decPtr = &ds->buf[ds->read_pos];
+ copyable = Min(ds->buf_pos - ds->read_pos, dst_size);
+ memcpy(dst, decPtr, copyable); /* read msg length */
+ *dst_processed = copyable;
+ *src_processed = 0;
+ ds->read_pos += copyable;
+
+ if (ds->read_pos == ds->buf_pos && RING_BUFFER_BYTES - ds->buf_pos < MESSAGE_MAX_BYTES)
+ {
+ ds->buf_pos = 0;
+ ds->read_pos = 0;
+ }
+ return ZS_OK;
+ }
+ decPtr = &ds->buf[ds->buf_pos];
+
+ decBytes = LZ4_decompress_safe_continue(ds->stream, src, decPtr, (int) src_size, RING_BUFFER_BYTES - ds->buf_pos);
+ if (decBytes < 0)
+ {
+ sprintf(last_error, "LZ4 decompression failed (src_size %ld, dst_size %ld, error: %d)", src_size, RING_BUFFER_BYTES - ds->buf_pos, decBytes);
+ ds->last_error = last_error;
+#ifndef FRONTEND
+ elog(ERROR, "%s", ds->last_error);
+#else
+ return ZS_DECOMPRESS_ERROR;
+#endif
+ }
+
+ copyable = Min(decBytes, dst_size);
+ memcpy(dst, decPtr, copyable);
+
+ *dst_processed = copyable;
+ *src_processed = src_size;
+
+ ds->buf_pos += decBytes;
+ ds->read_pos += copyable;
+ /* only reset the ring buffer after the internal buffer is drained */
+ if (copyable == decBytes && RING_BUFFER_BYTES - ds->buf_pos < MESSAGE_MAX_BYTES)
+ {
+ ds->buf_pos = 0;
+ ds->read_pos = 0;
+ }
+
+ return ZS_OK;
+}
+
+static bool
+lz4_decompress_buffered_data(void *d_stream)
+{
+ ZS_LZ4_DStream *ds = (ZS_LZ4_DStream *) d_stream;
+
+ return ds->read_pos < ds->buf_pos;
+}
+
+static int
+lz4_compress(void *c_stream, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed)
+{
+ ZS_LZ4_CStream *cs = (ZS_LZ4_CStream *) c_stream;
+ int cmpBytes;
+
+ src_size = Min(MESSAGE_MAX_BYTES, src_size);
+
+ memcpy((char *) (cs->buf) + cs->buf_pos, src, src_size); /* write msg length */
+
+ if (dst_size < LZ4_compressBound(src_size))
+ {
+ cs->last_error = "LZ4 compression failed: buffer not big enough";
+#ifndef FRONTEND
+ elog(ERROR, "%s", cs->last_error);
+#else
+ return ZS_COMPRESS_ERROR;
+#endif
+ }
+
+ cmpBytes = LZ4_compress_fast_continue(cs->stream, (char *) (cs->buf) + cs->buf_pos, dst, (int) src_size, (int) dst_size, cs->level);
+
+ if (cmpBytes < 0 || cmpBytes > MESSAGE_MAX_BYTES)
+ {
+ cs->last_error = "LZ4 compression failed";
+#ifndef FRONTEND
+ elog(ERROR, "%s", cs->last_error);
+#else
+ return ZS_COMPRESS_ERROR;
+#endif
+ }
+
+ *dst_processed = cmpBytes;
+ *src_processed = src_size;
+
+ cs->buf_pos += src_size;
+ if (cs->buf_pos >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES)
+ {
+ cs->buf_pos = 0;
+ }
+ return ZS_OK;
+}
+
+static bool
+lz4_compress_buffered_data(void *d_stream)
+{
+ return false;
+}
+
+
+static int
+lz4_end(void *c_stream, void *dst, size_t dst_size, size_t *dst_processed)
+{
+ *dst_processed = 0;
+ return ZS_OK;
+}
+
+static void
+lz4_free_compressor(void *c_stream)
+{
+ ZS_LZ4_CStream *cs = (ZS_LZ4_CStream *) c_stream;
+
+ if (cs != NULL)
+ {
+ if (cs->stream != NULL)
+ {
+ LZ4_freeStream(cs->stream);
+ }
+ FREE(cs);
+ }
+}
+
+static void
+lz4_free_decompressor(void *d_stream)
+{
+ ZS_LZ4_DStream *ds = (ZS_LZ4_DStream *) d_stream;
+
+ if (ds != NULL)
+ {
+ if (ds->stream != NULL)
+ {
+ LZ4_freeStreamDecode(ds->stream);
+ }
+ FREE(ds);
+ }
+}
+
+static char const *
+lz4_compress_error(void *stream)
+{
+ ZS_LZ4_CStream *cs = (ZS_LZ4_CStream *) stream;
+
+ /* lz4 doesn't have any explicit API to get the error names */
+ return cs->last_error;
+}
+
+static char const *
+lz4_decompress_error(void *stream)
+{
+ ZS_LZ4_DStream *ds = (ZS_LZ4_DStream *) stream;
+
+ /* lz4 doesn't have any explicit API to get the error names */
+ return ds->last_error;
+}
+
+static ZAlgorithm const lz4_algorithm = {
+ .algorithm = PG_COMPRESSION_LZ4,
+ .create_compressor = lz4_create_compressor,
+ .create_decompressor = lz4_create_decompressor,
+ .decompress = lz4_decompress,
+ .decompress_buffered_data = lz4_decompress_buffered_data,
+ .compress = lz4_compress,
+ .compress_buffered_data = lz4_compress_buffered_data,
+ .free_compressor = lz4_free_compressor,
+ .free_decompressor = lz4_free_decompressor,
+ .compress_error = lz4_compress_error,
+ .decompress_error = lz4_decompress_error,
+ .end_compression = lz4_end
+};
+
+#endif
+
+static const ZAlgorithm *
+zs_find_algorithm(pg_compress_algorithm algorithm)
+{
+#if HAVE_LIBZ
+ if (algorithm == PG_COMPRESSION_GZIP)
+ {
+ return &zlib_algorithm;
+ }
+#endif
+#if USE_LZ4
+ if (algorithm == PG_COMPRESSION_LZ4)
+ {
+ return &lz4_algorithm;
+ }
+#endif
+#if HAVE_LIBZSTD
+ if (algorithm == PG_COMPRESSION_ZSTD)
+ {
+ return &zstd_algorithm;
+ }
+#endif
+ return NULL;
+}
+
+static int
+zs_init_compressor(ZStream * zs, pg_compress_specification *spec)
+{
+ const ZAlgorithm *algorithm = zs_find_algorithm(spec->algorithm);
+
+ if (algorithm == NULL)
+ {
+ return -1;
+ }
+ zs->algorithm = algorithm;
+ zs->stream = zs->algorithm->create_compressor(spec->level);
+ if (zs->stream == NULL)
+ {
+ return -1;
+ }
+ return 0;
+}
+
+static int
+zs_init_decompressor(ZStream * zs, pg_compress_specification *spec)
+{
+ const ZAlgorithm *algorithm = zs_find_algorithm(spec->algorithm);
+
+ if (algorithm == NULL)
+ {
+ return -1;
+ }
+ zs->algorithm = algorithm;
+ zs->stream = zs->algorithm->create_decompressor();
+ if (zs->stream == NULL)
+ {
+ return -1;
+ }
+ return 0;
+}
+
+ZStream *
+zs_create_compressor(pg_compress_specification *spec)
+{
+ ZStream *zs = (ZStream *) ALLOC(sizeof(ZStream));
+
+ if (zs_init_compressor(zs, spec))
+ {
+ FREE(zs);
+ return NULL;
+ }
+
+ return zs;
+}
+
+ZStream *
+zs_create_decompressor(pg_compress_specification *spec)
+{
+ ZStream *zs = (ZStream *) ALLOC(sizeof(ZStream));
+
+ if (zs_init_decompressor(zs, spec))
+ {
+ FREE(zs);
+ return NULL;
+ }
+
+ return zs;
+}
+
+int
+zs_read(ZStream * zs, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed)
+{
+ int rc;
+
+ *src_processed = 0;
+ *dst_processed = 0;
+
+ rc = zs->algorithm->decompress(zs->stream,
+ src, src_size, src_processed,
+ dst, dst_size, dst_processed);
+
+ if (rc == ZS_OK || rc == ZS_INCOMPLETE_SRC || rc == ZS_STREAM_END)
+ {
+ return rc;
+ }
+
+ return ZS_DECOMPRESS_ERROR;
+}
+
+int
+zs_write(ZStream * zs, void const *buf, size_t size, size_t *processed, void *dst, size_t dst_size, size_t *dst_processed)
+{
+ int rc;
+
+ *processed = 0;
+ *dst_processed = 0;
+
+ rc = zs->algorithm->compress(zs->stream,
+ buf, size, processed,
+ dst, dst_size, dst_processed);
+
+ if (rc != ZS_OK)
+ {
+ return ZS_COMPRESS_ERROR;
+ }
+
+ return rc;
+}
+
+void
+zs_compressor_free(ZStream * zs)
+{
+ if (zs == NULL)
+ {
+ return;
+ }
+
+ if (zs->stream)
+ {
+ zs->algorithm->free_compressor(zs->stream);
+ }
+
+ FREE(zs);
+}
+
+void
+zs_decompressor_free(ZStream * zs)
+{
+ if (zs == NULL)
+ {
+ return;
+ }
+
+ if (zs->stream)
+ {
+ zs->algorithm->free_decompressor(zs->stream);
+ }
+
+ FREE(zs);
+}
+
+int
+zs_end_compression(ZStream * zs, void *dst, size_t dst_size, size_t *dst_processed)
+{
+ int rc;
+
+ *dst_processed = 0;
+
+ rc = zs->algorithm->end_compression(zs->stream, dst, dst_size, dst_processed);
+
+ if (rc != ZS_OK)
+ {
+ return ZS_COMPRESS_ERROR;
+ }
+
+ return rc;
+}
+
+char const *
+zs_compress_error(ZStream * zs)
+{
+ return zs->algorithm->compress_error(zs->stream);
+}
+
+char const *
+zs_decompress_error(ZStream * zs)
+{
+ return zs->algorithm->decompress_error(zs->stream);
+}
+
+bool
+zs_compress_buffered(ZStream * zs)
+{
+ return zs ? zs->algorithm->compress_buffered_data(zs->stream) : false;
+}
+
+bool
+zs_decompress_buffered(ZStream * zs)
+{
+ return zs ? zs->algorithm->decompress_buffered_data(zs->stream) : false;
+}
+
+pg_compress_algorithm
+zs_algorithm(ZStream * zs)
+{
+ return zs ? zs->algorithm->algorithm : PG_COMPRESSION_NONE;
+}
diff --git a/src/common/zpq_stream.c b/src/common/zpq_stream.c
new file mode 100644
index 0000000000..03ca8b7415
--- /dev/null
+++ b/src/common/zpq_stream.c
@@ -0,0 +1,1013 @@
+/*-------------------------------------------------------------------------
+ *
+ * zpq_stream.c
+ * IO stream layer applying ZStream compression to libpq
+ *
+ * Copyright (c) 2018-2023, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/common/zpq_stream.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FRONTEND
+#include "postgres.h"
+#else
+#include "postgres_fe.h"
+#endif
+#include <unistd.h>
+#include <math.h>
+
+#include "common/zpq_stream.h"
+
+#include <common/io_stream.h>
+#include <libpq/protocol.h>
+
+#include "pg_config.h"
+#include "port/pg_bswap.h"
+
+/* log warnings on backend */
+#ifndef FRONTEND
+#define pg_log_warning(...) elog(WARNING, __VA_ARGS__)
+#else
+#define pg_log_warning(...) (void)0
+#endif
+
+#ifndef FRONTEND
+#include "utils/memutils.h"
+#define ALLOC(size) MemoryContextAlloc(TopMemoryContext, size)
+#define STRDUP(str) pstrdup(str)
+#define FREE(size) pfree(size)
+#else
+#define ALLOC(size) malloc(size)
+#define STRDUP(str) strdup(str)
+#define FREE(size) free(size)
+#endif
+
+/* ZpqBuffer size, in bytes */
+#define ZPQ_BUFFER_SIZE 8192000
+
+#define ZPQ_COMPRESS_THRESHOLD 60
+
+/* startup messages have no type field and therefore have a null first byte */
+#define MESSAGE_TYPE_OFFSET(msg_type) (msg_type == '\0' ? 0 : 1)
+
+typedef struct ZpqBuffer ZpqBuffer;
+
+
+/* ZpqBuffer used as RX/TX buffer in ZpqStream */
+struct ZpqBuffer
+{
+ char buf[ZPQ_BUFFER_SIZE];
+ size_t size; /* current size of buf */
+ size_t pos; /* current position in buf, in range [0, size] */
+};
+
+/*
+ * Write up to "src_size" raw (decompressed) bytes.
+ * Returns number of written raw bytes or error code.
+ * Error code is either ZPQ_COMPRESS_ERROR or error code returned by the tx function.
+ * In the last case number of bytes written is stored in *src_processed.
+ */
+static int zpq_write(IoStreamLayer * layer, ZpqStream * zpq, void const *src, size_t src_size, size_t *bytes_written);
+
+/*
+ * Read up to "dst_size" raw (decompressed) bytes.
+ * Returns number of decompressed bytes or error code.
+ * Error code is either ZPQ_DECOMPRESS_ERROR or error code returned by the rx function.
+ */
+static ssize_t zpq_read(IoStreamLayer * layer, ZpqStream * zpq, void *dst, size_t dst_size, bool buffered_only);
+
+/*
+ * Return true if non-flushed data left in internal rx decompression buffer.
+ */
+static bool zpq_buffered_rx(ZpqStream * zpq);
+
+/*
+ * Return true if non-flushed data left in internal tx compression buffer.
+ */
+static bool zpq_buffered_tx(ZpqStream * zpq);
+
+/*
+ * Clear inflight messages and tracked state
+ */
+static void zpq_reset_write_state(ZpqStream * zpq);
+
+/*
+ * Free stream created by zs_create function.
+ */
+static void zpq_free(ZpqStream * zpq);
+
+IoStreamProcessor zpq_processor = {
+ .read = (io_stream_read_func) zpq_read,
+ .write = (io_stream_write_func) zpq_write,
+ .buffered_read_data = (io_stream_predicate) zpq_buffered_rx,
+ .buffered_write_data = (io_stream_predicate) zpq_buffered_tx,
+ .reset_write_state = (io_stream_consumer) zpq_reset_write_state,
+ .destroy = (io_stream_consumer) zpq_free
+};
+
+static inline void
+zpq_buf_init(ZpqBuffer * zb)
+{
+ zb->size = 0;
+ zb->pos = 0;
+}
+
+static inline size_t
+zpq_buf_left(ZpqBuffer * zb)
+{
+ Assert(zb->buf);
+ return ZPQ_BUFFER_SIZE - zb->size;
+}
+
+static inline size_t
+zpq_buf_unread(ZpqBuffer * zb)
+{
+ return zb->size - zb->pos;
+}
+
+static inline char *
+zpq_buf_size(ZpqBuffer * zb)
+{
+ return (char *) (zb->buf) + zb->size;
+}
+
+static inline char *
+zpq_buf_pos(ZpqBuffer * zb)
+{
+ return (char *) (zb->buf) + zb->pos;
+}
+
+static inline void
+zpq_buf_size_advance(ZpqBuffer * zb, size_t value)
+{
+ zb->size += value;
+}
+
+static inline void
+zpq_buf_pos_advance(ZpqBuffer * zb, size_t value)
+{
+ zb->pos += value;
+}
+
+static inline void
+zpq_buf_reuse(ZpqBuffer * zb)
+{
+ size_t unread = zpq_buf_unread(zb);
+
+ if (unread > 5) /* can read message header, don't do anything */
+ return;
+ if (unread == 0)
+ {
+ zb->size = 0;
+ zb->pos = 0;
+ return;
+ }
+ memmove(zb->buf, zb->buf + zb->pos, unread);
+ zb->size = unread;
+ zb->pos = 0;
+}
+
+struct ZpqStream
+{
+ ZStream *c_stream; /* underlying compression stream */
+ ZStream *d_stream; /* underlying decompression stream */
+
+ bool is_compressing; /* current compression state */
+
+ bool is_decompressing; /* current decompression state */
+ bool reading_compressed_header; /* compression header processing
+ * incomplete */
+ size_t rx_msg_bytes_left; /* number of bytes left to process without
+ * changing the decompression state */
+ size_t tx_msg_bytes_left; /* number of bytes left to process without
+ * changing the compression state */
+
+ ZpqBuffer rx_in; /* buffer for unprocessed data read from next
+ * stream layer */
+ ZpqBuffer tx_in; /* buffer for unprocessed data consumed by
+ * zpq_write */
+ ZpqBuffer tx_out; /* buffer for processed data waiting for send
+ * to next stream layer */
+
+ pg_compress_specification *compressors; /* compressors array holds the
+ * available compressors to use
+ * for compression/decompression */
+ size_t n_compressors; /* size of the compressors array */
+ pg_compress_algorithm compress_algs[COMPRESSION_ALGORITHM_COUNT - 1]; /* array of compression
+ * algorithms supported
+ * by the reciever of
+ * the stream */
+ pg_compress_algorithm compress_alg; /* active compression algorithm */
+ bool restart_compression; /* restart the compression stream
+ * after a client request */
+ bool first_compressed_message; /* track if we have sent the new
+ * compression algorithm */
+ pg_compress_algorithm decompress_alg; /* active decompression algorithm */
+};
+
+/*
+ * Choose the algorithm to use for the message of msg_type with msg_len.
+ * Returns a pg_compress_algorithm with a registered compressor, or PG_COMPRESSION_NONE if no compressor is appropriate
+ */
+static inline pg_compress_algorithm
+zpq_choose_algorithm(ZpqStream * zpq, char msg_type, uint32 msg_len)
+{
+ /*
+ * in theory we could choose the algorithm based on the message type
+ * and/or more complex heuristics, but at least for now we will just use
+ * the first available algorithm (which defaults to none if compression
+ * has not yet been enabled) for message types that would most obviously
+ * benefit from compression
+ */
+ if (msg_len >= ZPQ_COMPRESS_THRESHOLD && (msg_type == PqMsg_CopyData || msg_type == PqMsg_DataRow || msg_type == PqMsg_Query))
+ {
+ return zpq->compress_algs[0];
+ }
+ return PG_COMPRESSION_NONE;
+}
+
+static inline bool
+zpq_should_compress(ZpqStream * zpq, char msg_type, uint32 msg_len)
+{
+ return zpq_choose_algorithm(zpq, msg_type, msg_len) != PG_COMPRESSION_NONE;
+}
+
+static inline pg_compress_specification *
+zpq_find_compressor(ZpqStream * zpq, pg_compress_algorithm algorithm)
+{
+ int i;
+
+ for (i = 0; i < zpq->n_compressors; i++)
+ {
+ if (algorithm == zpq->compressors[i].algorithm)
+ return &zpq->compressors[i];
+ }
+ return NULL;
+}
+
+static inline bool
+zpq_is_compressed_msg(char msg_type)
+{
+ return msg_type == PqMsg_CompressedMessage;
+}
+
+ZpqStream *
+zpq_create(pg_compress_specification *compressors, size_t n_compressors, IoStream * stream)
+{
+ ZpqStream *zpq;
+
+ /* zpqStream needs at least one compressor */
+ if (n_compressors == 0 || compressors == NULL)
+ {
+ return NULL;
+ }
+ zpq = (ZpqStream *) ALLOC(sizeof(ZpqStream));
+ /* almost all fields should default to 0/NULL/PG_COMPRESSION_NONE */
+ memset(zpq, 0, sizeof(ZpqStream));
+ zpq->compressors = compressors;
+ zpq->n_compressors = n_compressors;
+ zpq_buf_init(&zpq->tx_in);
+ zpq_buf_init(&zpq->rx_in);
+ zpq_buf_init(&zpq->tx_out);
+
+ io_stream_add_layer(stream, &zpq_processor, zpq);
+
+ return zpq;
+}
+
+void
+zpq_enable_compression(ZpqStream * zpq, pg_compress_algorithm *algorithms, size_t n_algorithms)
+{
+ Assert(n_algorithms < COMPRESSION_ALGORITHM_COUNT);
+
+ for (int i = 0; i < COMPRESSION_ALGORITHM_COUNT - 1; i++)
+ {
+ if (i < n_algorithms)
+ {
+ zpq->compress_algs[i] = algorithms[i];
+ }
+ else
+ {
+ zpq->compress_algs[i] = PG_COMPRESSION_NONE;
+ }
+ }
+ zpq->restart_compression = true;
+}
+
+/* Compress up to src_size bytes from *src into CompressedData and write it to the tx buffer.
+ * Returns ZS_OK on success, ZS_COMPRESS_ERROR if encountered a compression error. */
+static inline int
+zpq_write_compressed_message(ZpqStream * zpq, char const *src, size_t src_size, size_t *src_processed)
+{
+ size_t compressed_len;
+ ssize_t rc;
+ uint32 size;
+ uint8 algorithm;
+
+ /* check if have enough space */
+ if (zpq_buf_left(&zpq->tx_out) <= 6)
+ {
+ /* too little space for CompressedData, abort */
+ *src_processed = 0;
+ return ZS_OK;
+ }
+
+ compressed_len = 0;
+ rc = zs_write(zpq->c_stream, src, src_size, src_processed,
+ zpq_buf_size(&zpq->tx_out) + 6, zpq_buf_left(&zpq->tx_out) - 6, &compressed_len);
+
+ if (compressed_len > 0)
+ {
+ /* write CompressedData type */
+ *zpq_buf_size(&zpq->tx_out) = PqMsg_CompressedMessage;
+ size = pg_hton32(compressed_len + 5);
+
+ memcpy(zpq_buf_size(&zpq->tx_out) + 1, &size, sizeof(uint32)); /* write msg length */
+ compressed_len += 6; /* append header length to compressed data
+ * length */
+ algorithm = zpq->first_compressed_message ? zpq->compress_alg : 0;
+ zpq->first_compressed_message = false;
+ memcpy(zpq_buf_size(&zpq->tx_out) + 5, &algorithm, sizeof(uint8)); /* write msg algorithm */
+ }
+
+ zpq_buf_size_advance(&zpq->tx_out, compressed_len);
+ return rc;
+}
+
+/* Copy the data directly from *src to the tx buffer */
+static void
+zpq_write_uncompressed(ZpqStream * zpq, char const *src, size_t src_size, size_t *src_processed)
+{
+ src_size = Min(zpq_buf_left(&zpq->tx_out), src_size);
+ memcpy(zpq_buf_size(&zpq->tx_out), src, src_size);
+
+ zpq_buf_size_advance(&zpq->tx_out, src_size);
+ *src_processed = src_size;
+}
+
+/* Determine if should compress the next message and change the current compression state */
+static int
+zpq_toggle_compression(ZpqStream * zpq, char msg_type, uint32 msg_len)
+{
+ pg_compress_algorithm new_compress_alg = zpq_choose_algorithm(zpq, msg_type, msg_len);
+ pg_compress_specification *spec;
+ bool should_compress = new_compress_alg != PG_COMPRESSION_NONE;
+
+ if (should_compress)
+ {
+ /*
+ * if the new compressor does not match the current one, change out
+ * the underlying z_stream
+ */
+ if (zpq->compress_alg != new_compress_alg || zpq->restart_compression)
+ {
+ zpq->restart_compression = false;
+ zpq->first_compressed_message = true;
+ zs_compressor_free(zpq->c_stream);
+ spec = zpq_find_compressor(zpq, new_compress_alg);
+ if (spec == NULL)
+ {
+ return ZPQ_FATAL_ERROR;
+ }
+ zpq->c_stream = zs_create_compressor(spec);
+ if (zpq->c_stream == NULL)
+ {
+ return ZPQ_FATAL_ERROR;
+ }
+ zpq->compress_alg = new_compress_alg;
+ }
+ }
+
+ zpq->is_compressing = should_compress;
+ zpq->tx_msg_bytes_left = msg_len + MESSAGE_TYPE_OFFSET(msg_type);
+ return 0;
+}
+
+/*
+ * Internal write function. Reads the data from *src buffer,
+ * determines the postgres messages type and length.
+ * If message matches the compression criteria, it wraps the message into
+ * CompressedData. Otherwise, leaves the message unchanged.
+ * If *src data ends with incomplete message header, this function is not
+ * going to read this message header.
+ * Returns 0 on success or error code
+ * Number of bytes written is stored in *processed.
+ */
+static int
+zpq_write_internal(ZpqStream * zpq, void const *src, size_t src_size, size_t *processed)
+{
+ size_t src_pos = 0;
+ ssize_t rc;
+
+ do
+ {
+ /*
+ * try to read ahead the next message types and increase
+ * tx_msg_bytes_left, if possible
+ */
+ while (zpq->tx_msg_bytes_left > 0 && src_size - src_pos >= zpq->tx_msg_bytes_left + 5)
+ {
+ char msg_type = *((char *) src + src_pos + zpq->tx_msg_bytes_left);
+ uint32 msg_len;
+
+ memcpy(&msg_len, (char *) src + src_pos + zpq->tx_msg_bytes_left + MESSAGE_TYPE_OFFSET(msg_type), 4);
+ msg_len = pg_ntoh32(msg_len);
+ if (zpq_should_compress(zpq, msg_type, msg_len) != zpq->is_compressing)
+ {
+ /*
+ * cannot proceed further, encountered compression toggle
+ * point
+ */
+ break;
+ }
+ zpq->tx_msg_bytes_left += msg_len + MESSAGE_TYPE_OFFSET(msg_type);
+ }
+
+ /*
+ * Write CompressedData if currently is compressing or have some
+ * buffered data left in underlying compression stream
+ */
+ if (zs_compress_buffered(zpq->c_stream) || (zpq->is_compressing && zpq->tx_msg_bytes_left > 0))
+ {
+ size_t buf_processed = 0;
+ size_t to_compress = Min(zpq->tx_msg_bytes_left, src_size - src_pos);
+
+ rc = zpq_write_compressed_message(zpq, (char *) src + src_pos, to_compress, &buf_processed);
+ src_pos += buf_processed;
+ zpq->tx_msg_bytes_left -= buf_processed;
+
+ if (rc != ZS_OK)
+ {
+ *processed = src_pos;
+ return rc;
+ }
+ }
+
+ /*
+ * If not going to compress the data from *src, just write it
+ * uncompressed.
+ */
+ else if (zpq->tx_msg_bytes_left > 0)
+ { /* determine next message type */
+ size_t copy_len = Min(src_size - src_pos, zpq->tx_msg_bytes_left);
+ size_t copy_processed = 0;
+
+ zpq_write_uncompressed(zpq, (char *) src + src_pos, copy_len, ©_processed);
+ src_pos += copy_processed;
+ zpq->tx_msg_bytes_left -= copy_processed;
+ }
+
+ /*
+ * Reached the compression toggle point, fetch next message header to
+ * determine compression state.
+ */
+ else
+ {
+ char msg_type;
+ uint32 msg_len;
+
+ if (src_size - src_pos < 5)
+ {
+ /*
+ * must return here because we can't continue without full
+ * message header
+ */
+ *processed = src_pos;
+ return 0;
+ }
+
+ msg_type = *((char *) src + src_pos);
+ memcpy(&msg_len, (char *) src + src_pos + MESSAGE_TYPE_OFFSET(msg_type), 4);
+ msg_len = pg_ntoh32(msg_len);
+ rc = zpq_toggle_compression(zpq, msg_type, msg_len);
+ if (rc)
+ {
+ *processed = src_pos;
+ return rc;
+ }
+ }
+
+ /*
+ * repeat sending while there is some data in input or internal
+ * compression buffer
+ */
+ } while (src_pos < src_size && zpq_buf_left(&zpq->tx_out) > 6);
+
+ *processed = src_pos;
+ return 0;
+}
+
+int
+zpq_write(IoStreamLayer * self, ZpqStream * zpq, void const *src, size_t src_size, size_t *bytes_written)
+{
+ size_t src_pos = 0;
+ ssize_t rc;
+
+ /* try to process as much data as possible before calling the tx_func */
+ while (zpq_buf_left(&zpq->tx_out) > 6)
+ {
+ size_t copy_len = Min(zpq_buf_left(&zpq->tx_in), src_size - src_pos);
+ size_t processed;
+
+ memcpy(zpq_buf_size(&zpq->tx_in), (char *) src + src_pos, copy_len);
+ zpq_buf_size_advance(&zpq->tx_in, copy_len);
+ src_pos += copy_len;
+
+ if (zpq_buf_unread(&zpq->tx_in) == 0 && !zs_compress_buffered(zpq->c_stream))
+ {
+ break;
+ }
+
+ processed = 0;
+
+ rc = zpq_write_internal(zpq, zpq_buf_pos(&zpq->tx_in), zpq_buf_unread(&zpq->tx_in), &processed);
+ zpq_buf_pos_advance(&zpq->tx_in, processed);
+ zpq_buf_reuse(&zpq->tx_in);
+ if (rc < 0)
+ {
+ *bytes_written = src_pos;
+ return rc;
+ }
+ if (processed == 0)
+ {
+ break;
+ }
+ }
+ *bytes_written = src_pos;
+
+ /*
+ * call the tx_func if have any bytes to send
+ */
+ while (zpq_buf_unread(&zpq->tx_out))
+ {
+ size_t count;
+
+ rc = io_stream_next_write(self, zpq_buf_pos(&zpq->tx_out), zpq_buf_unread(&zpq->tx_out), &count);
+ if (!rc && count > 0)
+ {
+ zpq_buf_pos_advance(&zpq->tx_out, count);
+ }
+ else
+ {
+ zpq_buf_reuse(&zpq->tx_out);
+ return rc;
+ }
+ }
+
+ zpq_buf_reuse(&zpq->tx_out);
+ return 0;
+}
+
+/* Decompress bytes from RX buffer and write up to dst_len of uncompressed data to *dst.
+ * Returns:
+ * ZS_OK on success,
+ * ZS_STREAM_END if reached end of compressed chunk
+ * ZS_DECOMPRESS_ERROR if encountered a decompression error */
+static inline ssize_t
+zpq_read_compressed_message(ZpqStream * zpq, char *dst, size_t dst_len, size_t *dst_processed)
+{
+ size_t rx_processed = 0;
+ ssize_t rc;
+ size_t read_len = Min(zpq->rx_msg_bytes_left, zpq_buf_unread(&zpq->rx_in));
+
+ Assert(read_len == zpq->rx_msg_bytes_left);
+ rc = zs_read(zpq->d_stream, zpq_buf_pos(&zpq->rx_in), read_len, &rx_processed,
+ dst, dst_len, dst_processed);
+
+ zpq_buf_pos_advance(&zpq->rx_in, rx_processed);
+ zpq->rx_msg_bytes_left -= rx_processed;
+ return rc;
+}
+
+/* Copy up to dst_len bytes from rx buffer to *dst.
+ * Returns amount of bytes copied. */
+static inline size_t
+zpq_read_uncompressed(ZpqStream * zpq, char *dst, size_t dst_len)
+{
+ size_t copy_len;
+
+ Assert(zpq_buf_unread(&zpq->rx_in) > 0);
+ copy_len = Min(zpq->rx_msg_bytes_left, Min(zpq_buf_unread(&zpq->rx_in), dst_len));
+
+ memcpy(dst, zpq_buf_pos(&zpq->rx_in), copy_len);
+
+ zpq_buf_pos_advance(&zpq->rx_in, copy_len);
+ zpq->rx_msg_bytes_left -= copy_len;
+ return copy_len;
+}
+
+/* Determine if should decompress the next message and
+ * change the current decompression state */
+static inline void
+zpq_toggle_decompression(ZpqStream * zpq)
+{
+ uint32 msg_len;
+ char msg_type = *zpq_buf_pos(&zpq->rx_in);
+
+ memcpy(&msg_len, zpq_buf_pos(&zpq->rx_in) + MESSAGE_TYPE_OFFSET(msg_type), 4);
+ msg_len = pg_ntoh32(msg_len);
+
+ zpq->is_decompressing = zpq_is_compressed_msg(msg_type);
+ zpq->rx_msg_bytes_left = msg_len + MESSAGE_TYPE_OFFSET(msg_type);
+
+ if (zpq->is_decompressing)
+ {
+ /* compressed message header is no longer needed, just skip it */
+ zpq_buf_pos_advance(&zpq->rx_in, 5);
+ zpq->rx_msg_bytes_left -= 5;
+ zpq->reading_compressed_header = true;
+ }
+}
+
+static inline ssize_t
+zpq_process_switch(ZpqStream * zpq)
+{
+ pg_compress_algorithm algorithm;
+ pg_compress_specification *spec;
+
+ if (zpq_buf_unread(&zpq->rx_in) < 1)
+ {
+ return 0;
+ }
+
+ algorithm = *zpq_buf_pos(&zpq->rx_in);
+
+ zpq_buf_pos_advance(&zpq->rx_in, 1);
+ zpq->reading_compressed_header = false;
+ zpq->rx_msg_bytes_left -= 1;
+
+ /*
+ * if the message specifies an algorithm, restart the decompressor with
+ * the new algorithm (can be the same as the previous one)
+ */
+ if (algorithm > 0)
+ {
+ zs_decompressor_free(zpq->d_stream);
+ spec = zpq_find_compressor(zpq, algorithm);
+ if (spec == NULL)
+ {
+ return ZPQ_FATAL_ERROR;
+ }
+ zpq->d_stream = zs_create_decompressor(spec);
+ if (zpq->d_stream == NULL)
+ {
+ return ZPQ_FATAL_ERROR;
+ }
+ zpq->decompress_alg = algorithm;
+ }
+
+ return 0;
+}
+
+ssize_t
+zpq_read(IoStreamLayer * self, ZpqStream * zpq, void *dst, size_t dst_size, bool buffered_only)
+{
+ size_t dst_pos = 0;
+ size_t dst_processed = 0;
+ ssize_t rc;
+
+ /* Read until some data fetched */
+ while (dst_pos == 0)
+ {
+ zpq_buf_reuse(&zpq->rx_in);
+
+ if (!zpq_buffered_rx(zpq) || (zpq->is_decompressing && zpq_buf_unread(&zpq->rx_in) < zpq->rx_msg_bytes_left))
+ {
+ rc = io_stream_next_read(self, zpq_buf_size(&zpq->rx_in), zpq_buf_left(&zpq->rx_in), buffered_only);
+ if (rc > 0) /* read fetches some data */
+ {
+ zpq_buf_size_advance(&zpq->rx_in, rc);
+ }
+ else if (rc == 0)
+ {
+ /* got no more data; return what we have */
+ return dst_pos;
+ }
+ else /* read failed */
+ {
+ return rc;
+ }
+ }
+
+ /*
+ * try to read ahead the next message types and increase
+ * rx_msg_bytes_left, if possible (ONLY UNCOMPRESSED MESSAGES)
+ */
+ while (!zpq->is_decompressing && zpq->rx_msg_bytes_left > 0 && (zpq_buf_unread(&zpq->rx_in) >= zpq->rx_msg_bytes_left + 5))
+ {
+ char msg_type;
+ uint32 msg_len;
+
+ msg_type = *(zpq_buf_pos(&zpq->rx_in) + zpq->rx_msg_bytes_left);
+ if (zpq_is_compressed_msg(msg_type))
+ {
+ /*
+ * cannot proceed further, encountered compression toggle
+ * point
+ */
+ break;
+ }
+
+ memcpy(&msg_len, zpq_buf_pos(&zpq->rx_in) + zpq->rx_msg_bytes_left + MESSAGE_TYPE_OFFSET(msg_type), 4);
+ zpq->rx_msg_bytes_left += pg_ntoh32(msg_len) + MESSAGE_TYPE_OFFSET(msg_type);
+ }
+
+
+ /*
+ * If we are in the middle of reading a message, keep reading it until
+ * we reach the end at which point we need to check if we should
+ * toggle compression
+ */
+ if (zpq->rx_msg_bytes_left > 0 || zs_decompress_buffered(zpq->d_stream))
+ {
+ dst_processed = 0;
+ if (zpq->is_decompressing || zs_decompress_buffered(zpq->d_stream))
+ {
+ if (zpq->reading_compressed_header)
+ {
+ zpq_process_switch(zpq);
+ }
+ if (!zs_decompress_buffered(zpq->d_stream) && zpq_buf_unread(&zpq->rx_in) < zpq->rx_msg_bytes_left)
+ {
+ /*
+ * prefer to read only the fully compressed messages or
+ * read if some data is buffered
+ */
+ continue;
+ }
+ rc = zpq_read_compressed_message(zpq, dst, dst_size - dst_pos, &dst_processed);
+ dst_pos += dst_processed;
+ if (rc == ZS_STREAM_END)
+ {
+ continue;
+ }
+ if (rc != ZS_OK)
+ {
+ return rc;
+ }
+ }
+ else
+ dst_pos += zpq_read_uncompressed(zpq, dst, dst_size - dst_pos);
+ }
+ else if (zpq_buf_unread(&zpq->rx_in) >= 5)
+ zpq_toggle_decompression(zpq);
+ }
+ return dst_pos;
+}
+
+bool
+zpq_buffered_rx(ZpqStream * zpq)
+{
+ return zpq ? zpq_buf_unread(&zpq->rx_in) >= 5 || (zpq_buf_unread(&zpq->rx_in) > 0 && zpq->rx_msg_bytes_left > 0) ||
+ zs_decompress_buffered(zpq->d_stream) : 0;
+}
+
+bool
+zpq_buffered_tx(ZpqStream * zpq)
+{
+ return zpq ? zpq_buf_unread(&zpq->tx_in) >= 5 || (zpq_buf_unread(&zpq->tx_in) > 0 && zpq->tx_msg_bytes_left > 0) || zpq_buf_unread(&zpq->tx_out) > 0 ||
+ zs_compress_buffered(zpq->c_stream) : 0;
+}
+
+void
+zpq_reset_write_state(ZpqStream * zpq)
+{
+ /*
+ * We need to flush where we are in the msg to ensure we keep our reads
+ * correctly aligned
+ */
+ zpq->tx_msg_bytes_left = 0;
+ zpq->tx_in.pos = 0;
+ zpq->tx_in.size = 0;
+ zpq->tx_out.pos = 0;
+ zpq->tx_out.size = 0;
+
+ /*
+ * Disable compression for the rest of the life of this connection since
+ * any existing state will be stale, and we only call this after a failure
+ * at which point the connection will be terminated shortly anyways
+ */
+ zpq->is_compressing = false;
+ for (int i = 0; i < COMPRESSION_ALGORITHM_COUNT - 1; i++)
+ {
+ zpq->compress_algs[i] = PG_COMPRESSION_NONE;
+ }
+}
+
+void
+zpq_free(ZpqStream * zpq)
+{
+ if (zpq)
+ {
+ if (zpq->c_stream)
+ {
+ zs_compressor_free(zpq->c_stream);
+ }
+ if (zpq->d_stream)
+ {
+ zs_decompressor_free(zpq->d_stream);
+ }
+ FREE(zpq);
+ }
+}
+
+char const *
+zpq_compress_error(ZpqStream * zpq)
+{
+ return zs_compress_error(zpq->c_stream);
+}
+
+char const *
+zpq_decompress_error(ZpqStream * zpq)
+{
+ return zs_decompress_error(zpq->d_stream);
+}
+
+pg_compress_algorithm
+zpq_compress_algorithm(ZpqStream * zpq)
+{
+ return zs_algorithm(zpq->c_stream);
+}
+
+pg_compress_algorithm
+zpq_decompress_algorithm(ZpqStream * zpq)
+{
+ return zs_algorithm(zpq->d_stream);
+}
+
+char *
+zpq_algorithms(ZpqStream * zpq)
+{
+ return zpq_serialize_compressors(zpq->compressors, zpq->n_compressors);
+}
+
+int
+zpq_parse_compression_setting(const char *val, pg_compress_specification *compressors, size_t *n_compressors)
+{
+ int i;
+
+ *n_compressors = 0;
+ memset(compressors, 0, sizeof(pg_compress_specification) * COMPRESSION_ALGORITHM_COUNT);
+
+ if (pg_strcasecmp(val, "true") == 0 ||
+ pg_strcasecmp(val, "yes") == 0 ||
+ pg_strcasecmp(val, "on") == 0 ||
+ pg_strcasecmp(val, "1") == 0)
+ {
+ int j = 0;
+
+ /*
+ * return all available compressors
+ */
+ for (i = 1; i < COMPRESSION_ALGORITHM_COUNT; i++)
+ {
+ if (supported_compression_algorithm(i))
+ *n_compressors += 1;
+ }
+
+ for (i = 1; i < COMPRESSION_ALGORITHM_COUNT; i++)
+ {
+ if (supported_compression_algorithm(i))
+ {
+ parse_compress_specification(i, NULL, &compressors[j], 0);
+ j += 1;
+ }
+ }
+ return 1;
+ }
+
+ if (*val == 0 ||
+ pg_strcasecmp(val, "false") == 0 ||
+ pg_strcasecmp(val, "no") == 0 ||
+ pg_strcasecmp(val, "off") == 0 ||
+ pg_strcasecmp(val, "0") == 0)
+ {
+ /* Compression is disabled */
+ return 0;
+ }
+
+ return zpq_deserialize_compressors(val, compressors, n_compressors) ? 1 : -1;
+}
+
+bool
+zpq_deserialize_compressors(char const *c_string, pg_compress_specification *compressors, size_t *n_compressors)
+{
+ int selected_alg_mask = 0; /* bitmask of already selected
+ * algorithms to avoid duplicates in
+ * compressors */
+ char *c_string_dup = STRDUP(c_string); /* following parsing can
+ * modify the string */
+ char *p = c_string_dup;
+
+ *n_compressors = 0;
+ memset(compressors, 0, sizeof(pg_compress_specification) * COMPRESSION_ALGORITHM_COUNT);
+
+ while (*p != '\0')
+ {
+ char *sep = strchr(p, ';');
+ char *col;
+ char *error_detail;
+ pg_compress_algorithm algorithm;
+ pg_compress_specification *spec = &compressors[*n_compressors];
+
+ if (sep != NULL)
+ *sep = '\0';
+
+ col = strchr(p, ':');
+ if (col != NULL)
+ {
+ *col = '\0';
+ }
+ if (!parse_compress_algorithm(p, &algorithm))
+ {
+ pg_log_warning("invalid compression algorithm %s", p);
+ goto error;
+ }
+
+ if (supported_compression_algorithm(algorithm))
+ {
+ parse_compress_specification(algorithm, col == NULL ? NULL : col + 1, spec, PG_COMPRESSION_OPTION_COMPRESS | PG_COMPRESSION_OPTION_DECOMPRESS);
+ error_detail = validate_compress_specification(spec);
+ if (error_detail)
+ {
+ pg_log_warning("invalid compression specification: %s", error_detail);
+ goto error;
+ }
+
+ if (algorithm == PG_COMPRESSION_NONE)
+ {
+ pg_log_warning("algorithm none is not valid for protocol compression");
+ goto error;
+ }
+
+ if (selected_alg_mask & (1 << algorithm))
+ {
+ /* duplicates are not allowed */
+ pg_log_warning("duplicate algorithm %s in compressors string %s", get_compress_algorithm_name(algorithm), c_string);
+ goto error;
+ }
+
+ *n_compressors += 1;
+ selected_alg_mask |= 1 << algorithm;
+ }
+ else
+ {
+ /*
+ * Intentionally do not return an error, as we must support
+ * clients/servers parsing algorithms they don't suppport mixed
+ * with ones they do support
+ */
+ pg_log_warning("this build does not support compression with %s",
+ get_compress_algorithm_name(algorithm));
+ }
+
+ if (sep)
+ p = sep + 1;
+ else
+ break;
+ }
+
+ FREE(c_string_dup);
+ return true;
+
+error:
+ FREE(c_string_dup);
+ *n_compressors = 0;
+ return false;
+}
+
+char *
+zpq_serialize_compressors(pg_compress_specification const *compressors, size_t n_compressors)
+{
+ char *res;
+ char *p;
+ size_t i;
+ size_t total_len = 0;
+
+ if (n_compressors == 0)
+ {
+ return NULL;
+ }
+
+ for (i = 0; i < n_compressors; i++)
+ {
+ /*
+ * single entry looks like "alg_name:compression_specification," so +2
+ * is for ":" and ";" symbols (or trailing null)
+ */
+ total_len += strlen(get_compress_algorithm_name(compressors[i].algorithm)) + serialize_compress_specification(&compressors[i], NULL, 0) + 2;
+ }
+
+ res = p = ALLOC(total_len);
+
+ for (i = 0; i < n_compressors; i++)
+ {
+ p += sprintf(p, "%s:", get_compress_algorithm_name(compressors[i].algorithm));
+ p += serialize_compress_specification(&compressors[i], p, total_len - (p - res));
+ if (i < n_compressors - 1)
+ *p++ = ';';
+ }
+ return res;
+}
diff --git a/src/include/common/compression.h b/src/include/common/compression.h
index c94ace6e8a..86ff9815cf 100644
--- a/src/include/common/compression.h
+++ b/src/include/common/compression.h
@@ -17,6 +17,7 @@
/*
* These values are stored in disk, for example in files generated by pg_dump.
* Create the necessary backwards compatibility layers if their order changes.
+ * Make sure to keep COMPRESSION_ALGORITHM_COUNT in sync if adding new values.
*/
typedef enum pg_compress_algorithm
{
@@ -26,8 +27,12 @@ typedef enum pg_compress_algorithm
PG_COMPRESSION_ZSTD,
} pg_compress_algorithm;
+#define COMPRESSION_ALGORITHM_COUNT (PG_COMPRESSION_ZSTD + 1)
+
#define PG_COMPRESSION_OPTION_WORKERS (1 << 0)
#define PG_COMPRESSION_OPTION_LONG_DISTANCE (1 << 1)
+#define PG_COMPRESSION_OPTION_COMPRESS (1 << 2)
+#define PG_COMPRESSION_OPTION_DECOMPRESS (1 << 3)
typedef struct pg_compress_specification
{
@@ -36,7 +41,11 @@ typedef struct pg_compress_specification
int level;
int workers;
bool long_distance;
- char *parse_error; /* NULL if parsing was OK, else message */
+ bool compress;
+ bool decompress;
+ bool has_error;
+ char parse_error[255]; /* Populated with error message if
+ * has_error is true */
} pg_compress_specification;
extern void parse_compress_options(const char *option, char **algorithm,
@@ -46,8 +55,13 @@ extern const char *get_compress_algorithm_name(pg_compress_algorithm algorithm);
extern void parse_compress_specification(pg_compress_algorithm algorithm,
char *specification,
- pg_compress_specification *result);
+ pg_compress_specification *result,
+ unsigned allowed_options);
+extern int serialize_compress_specification(const pg_compress_specification *spec,
+ char *buffer,
+ size_t buffer_size);
extern char *validate_compress_specification(pg_compress_specification *);
+extern bool supported_compression_algorithm(pg_compress_algorithm algorithm);
#endif
diff --git a/src/include/common/io_stream.h b/src/include/common/io_stream.h
index 9af88aa9ea..77b8af71d6 100644
--- a/src/include/common/io_stream.h
+++ b/src/include/common/io_stream.h
@@ -23,7 +23,7 @@ typedef struct IoStream IoStream;
typedef ssize_t (*io_stream_read_func) (IoStreamLayer * self, void *context, void *data, size_t size, bool buffered_only);
typedef int (*io_stream_write_func) (IoStreamLayer * self, void *context, void const *data, size_t size, size_t *bytes_written);
typedef bool (*io_stream_predicate) (void *context);
-typedef void (*io_stream_destroy_func) (void *context);
+typedef void (*io_stream_consumer) (void *context);
typedef struct IoStreamProcessor
{
@@ -49,11 +49,18 @@ typedef struct IoStreamProcessor
*/
io_stream_predicate buffered_write_data;
+ /*
+ * Optional Will be called if data being sent to the stream has been reset
+ * early (e.g. due to a transmission error). Only necessary if the
+ * processor inspects the messages of the stream and tracks related state
+ */
+ io_stream_consumer reset_write_state;
+
/*
* Optional will be called as part of io_stream_destroy when cleaning up
* the stream
*/
- io_stream_destroy_func destroy;
+ io_stream_consumer destroy;
} IoStreamProcessor;
/*
@@ -108,6 +115,12 @@ extern bool io_stream_buffered_read_data(IoStream * stream);
*/
extern bool io_stream_buffered_write_data(IoStream * stream);
+/*
+ * Resets any state in the processors about currently in-flight messages
+ * Should be called if message transmission is aborted for any reason
+ */
+extern void io_stream_reset_write_state(IoStream * stream);
+
/*
* Read data from the next layer of the stream
* (to be used by io_stream_read_func)
diff --git a/src/include/common/z_stream.h b/src/include/common/z_stream.h
new file mode 100644
index 0000000000..e456d509e1
--- /dev/null
+++ b/src/include/common/z_stream.h
@@ -0,0 +1,98 @@
+/*-------------------------------------------------------------------------
+ *
+ * z_stream.h
+ * Streaming compression algorithms
+ *
+ * Copyright (c) 2018-2023, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/common/z_stream.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef Z_STREAM_H
+#define Z_STREAM_H
+
+#include <stdlib.h>
+
+#include "compression.h"
+
+#define ZS_OK (0)
+#define ZS_IO_ERROR (-1)
+#define ZS_DECOMPRESS_ERROR (-2)
+#define ZS_COMPRESS_ERROR (-3)
+#define ZS_STREAM_END (-4)
+#define ZS_INCOMPLETE_SRC (-5) /* cannot decompress unless full src message
+ * is fetched */
+
+struct ZStream;
+typedef struct ZStream ZStream;
+
+#endif
+
+/*
+ * Create compression stream for sending compressed data.
+ * spec: specifications of chosen decompression algorithm
+ */
+extern ZStream * zs_create_compressor(pg_compress_specification *spec);
+
+/*
+ * Create decompression stream for reading compressed data.
+ * spec: specifications of chosen decompression algorithm
+ */
+extern ZStream * zs_create_decompressor(pg_compress_specification *spec);
+
+/*
+ * Read up to "size" raw (decompressed) bytes.
+ * Returns ZS_OK on success or error code.
+ * Stores bytes read from src in src_processed, bytes written to dst in dst_process.
+ */
+extern int zs_read(ZStream * zs, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed);
+
+/*
+ * Write up to "size" raw (decompressed) bytes.
+ * Returns number of written raw bytes or error code.
+ * Returns ZS_OK on success or error code.
+ * Stores bytes read from buf in processed, bytes written to dst in dst_process
+ */
+extern int zs_write(ZStream * zs, void const *buf, size_t size, size_t *processed, void *dst, size_t dst_size, size_t *dst_processed);
+
+/*
+ * Returns if there is buffered raw data remaining in the stream to compress
+ */
+extern bool zs_compress_buffered(ZStream * zs);
+
+/*
+ * Returns if there is buffered decompressed data remaining in the stream to read
+ */
+extern bool zs_decompress_buffered(ZStream * zs);
+
+/*
+ * Get decompressor error message.
+ */
+extern char const *zs_decompress_error(ZStream * zs);
+
+/*
+ * Get compressor error message.
+ */
+extern char const *zs_compress_error(ZStream * zs);
+
+/*
+ * End the compression stream.
+ */
+extern int zs_end_compression(ZStream * zs, void *dst, size_t dst_size, size_t *dst_processed);
+
+/*
+ * Free stream created by zs_create_compressor function.
+ */
+extern void zs_compressor_free(ZStream * zs);
+
+/*
+ * Free stream created by zs_create_decompressor function.
+ */
+extern void zs_decompressor_free(ZStream * zs);
+
+/*
+ * Get the descriptor of chosen algorithm.
+ */
+extern pg_compress_algorithm zs_algorithm(ZStream * zs);
diff --git a/src/include/common/zpq_stream.h b/src/include/common/zpq_stream.h
new file mode 100644
index 0000000000..15570f2363
--- /dev/null
+++ b/src/include/common/zpq_stream.h
@@ -0,0 +1,91 @@
+/*-------------------------------------------------------------------------
+ *
+ * zpq_stream.h
+ * IO stream layer applying ZStream compression to libpq
+ *
+ * Copyright (c) 2018-2023, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/common/zpq_stream.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "io_stream.h"
+#include "z_stream.h"
+
+#ifndef ZPQ_STREAM_H
+#define ZPQ_STREAM_H
+
+#define ZPQ_FATAL_ERROR (-7)
+struct ZpqStream;
+typedef struct ZpqStream ZpqStream;
+
+#endif
+
+/*
+ * Create compression stream with rx/tx function for reading/sending compressed data.
+ * io_stream: IO Stream to layer on top of
+ * rx_data: received data (compressed data already fetched from input stream)
+ * rx_data_size: size of data fetched from input stream
+ * The returned ZpqStream can only be destroyed by destoing the IoStream with io_stream_destroy.
+ */
+extern ZpqStream * zpq_create(pg_compress_specification *compressors, size_t n_compressors, IoStream * stream);
+
+/*
+ * Start compressing applicable outgoing data once the connection is sufficiently set up
+ */
+extern void zpq_enable_compression(ZpqStream * zpq, pg_compress_algorithm *algorithms, size_t n_algorithms);
+
+/*
+ * Get decompressor error message.
+ */
+extern char const *zpq_decompress_error(ZpqStream * zpq);
+
+/*
+ * Get compressor error message.
+ */
+extern char const *zpq_compress_error(ZpqStream * zpq);
+
+/*
+ * Get the name of the current compression algorithm.
+ */
+extern pg_compress_algorithm zpq_compress_algorithm(ZpqStream * zpq);
+
+/*
+ * Get the name of the current decompression algorithm.
+ */
+extern pg_compress_algorithm zpq_decompress_algorithm(ZpqStream * zpq);
+
+/*
+ * Parse the compression setting.
+ * Compressors must be an array of length COMPRESSION_ALGORITHM_COUNT
+ * Returns:
+ * - 1 if the compression setting is valid
+ * - 0 if the compression setting is valid but disabled
+ * - -1 if the compression setting is invalid
+ * It also populates the compressors array with the recognized compressors. Size of the array is stored in n_compressors.
+ * If no supported compressors recognized or if compression is disabled, then n_compressors is set to 0.
+ */
+extern int
+ zpq_parse_compression_setting(const char *val, pg_compress_specification *compressors, size_t *n_compressors);
+
+/* Serialize the compressors array to string so it can be transmitted to the other side during the compression startup.
+ * For example, for array of two compressors (zstd, level 1), (zlib, level 2) resulting string would look like "zstd:1;zlib:2".
+ * Returns the resulting string.
+ */
+extern char
+ *zpq_serialize_compressors(pg_compress_specification const *compressors, size_t n_compressors);
+
+/* Deserialize the compressors string received during the compression setup to a compressors array.
+ * Compressors must be an array of length COMPRESSION_ALGORITHM_COUNT
+ * Returns:
+ * - true if the compressors string is successfully parsed
+ * - false otherwise
+ * It also populates the compressors array with the recognized compressors. Size of the array is stored in n_compressors.
+ * If no supported compressors are recognized or c_string is empty, then n_compressors is set to 0.
+ */
+bool
+ zpq_deserialize_compressors(char const *c_string, pg_compress_specification *compressors, size_t *n_compressors);
+
+/* Returns the currently enabled compression algorithms using zpq_serialize_compressors */
+char *zpq_algorithms(ZpqStream * zpq);
diff --git a/src/include/libpq/compression.h b/src/include/libpq/compression.h
new file mode 100644
index 0000000000..927ef42751
--- /dev/null
+++ b/src/include/libpq/compression.h
@@ -0,0 +1,30 @@
+/*-------------------------------------------------------------------------
+ *
+ * compression.h
+ * Interface to libpq/compression.c
+ *
+ *
+ * Copyright (c) 2023, PostgreSQL Global Development Group
+ *
+ * src/include/libpq/compression.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef LIBPQ_COMPRESSION_H
+#define LIBPQ_COMPRESSION_H
+
+#include "postgres.h"
+#include "libpq-be.h"
+#include "common/compression.h"
+
+extern PGDLLIMPORT char *libpq_compress_algorithms;
+extern PGDLLIMPORT pg_compress_specification libpq_compressors[COMPRESSION_ALGORITHM_COUNT];
+extern PGDLLIMPORT size_t libpq_n_compressors;
+
+/*
+ * Enbles compression processing on the given port.
+ * val is the value of the _pq_.libpq_compression startup packet parameter
+ */
+extern void configure_libpq_compression(Port *port, const char *val);
+
+#endif /* LIBPQ_COMPRESSION_H */
diff --git a/src/include/libpq/libpq-be-fe-helpers.h b/src/include/libpq/libpq-be-fe-helpers.h
index 41e3bb4376..cd840391f0 100644
--- a/src/include/libpq/libpq-be-fe-helpers.h
+++ b/src/include/libpq/libpq-be-fe-helpers.h
@@ -206,11 +206,14 @@ libpqsrv_connect_internal(PGconn *conn, uint32 wait_event_info)
else
io_flag = WL_SOCKET_WRITEABLE;
- rc = WaitLatchOrSocket(MyLatch,
- WL_EXIT_ON_PM_DEATH | WL_LATCH_SET | io_flag,
- PQsocket(conn),
- 0,
- wait_event_info);
+ if (PQreadPending(conn))
+ rc = WL_SOCKET_READABLE;
+ else
+ rc = WaitLatchOrSocket(MyLatch,
+ WL_EXIT_ON_PM_DEATH | WL_LATCH_SET | io_flag,
+ PQsocket(conn),
+ 0,
+ wait_event_info);
/* Interrupted? */
if (rc & WL_LATCH_SET)
diff --git a/src/include/libpq/libpq-be.h b/src/include/libpq/libpq-be.h
index 87ba7f5ea0..ec3b69351c 100644
--- a/src/include/libpq/libpq-be.h
+++ b/src/include/libpq/libpq-be.h
@@ -53,6 +53,8 @@ typedef struct
#endif
#endif /* ENABLE_SSPI */
+#include <common/zpq_stream.h>
+
#include "common/io_stream.h"
#include "datatype/timestamp.h"
#include "libpq/hba.h"
@@ -161,6 +163,7 @@ typedef struct Port
int remote_hostname_errcode; /* see above */
char *remote_port; /* text rep of remote port */
CAC_state canAcceptConnections; /* postmaster connection status */
+ ZpqStream *zpq_stream; /* streaming compression state */
/*
* Information that needs to be saved from the startup packet and passed
diff --git a/src/include/libpq/libpq.h b/src/include/libpq/libpq.h
index 3612280146..01cc3911fa 100644
--- a/src/include/libpq/libpq.h
+++ b/src/include/libpq/libpq.h
@@ -73,6 +73,7 @@ extern void StreamClose(pgsocket sock);
extern void TouchSocketFiles(void);
extern void RemoveSocketFiles(void);
extern void pq_init(void);
+extern int pq_configure(Port *port);
extern int pq_getbytes(char *s, size_t len);
extern void pq_startmsgread(void);
extern void pq_endmsgread(void);
diff --git a/src/include/libpq/protocol.h b/src/include/libpq/protocol.h
index cc46f4b586..d9eaebbf7e 100644
--- a/src/include/libpq/protocol.h
+++ b/src/include/libpq/protocol.h
@@ -63,7 +63,7 @@
#define PqMsg_CopyDone 'c'
#define PqMsg_CopyData 'd'
-
+#define PqMsg_CompressedMessage 'z'
/* These are the authentication request codes sent by the backend. */
diff --git a/src/include/utils/guc_hooks.h b/src/include/utils/guc_hooks.h
index 3d74483f44..ee34fbb20e 100644
--- a/src/include/utils/guc_hooks.h
+++ b/src/include/utils/guc_hooks.h
@@ -63,6 +63,8 @@ extern bool check_effective_io_concurrency(int *newval, void **extra,
GucSource source);
extern bool check_huge_page_size(int *newval, void **extra, GucSource source);
extern const char *show_in_hot_standby(void);
+extern bool check_libpq_compression(char **newval, void **extra, GucSource source);
+extern void assign_libpq_compression(const char *newval, void *extra);
extern bool check_locale_messages(char **newval, void **extra, GucSource source);
extern void assign_locale_messages(const char *newval, void *extra);
extern bool check_locale_monetary(char **newval, void **extra, GucSource source);
diff --git a/src/interfaces/libpq/exports.txt b/src/interfaces/libpq/exports.txt
index 850734ac96..392a6815e3 100644
--- a/src/interfaces/libpq/exports.txt
+++ b/src/interfaces/libpq/exports.txt
@@ -191,3 +191,6 @@ PQclosePrepared 188
PQclosePortal 189
PQsendClosePrepared 190
PQsendClosePortal 191
+PQcompression 192
+PQreadPending 193
+PQserverCompression 194
diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c
index a0f12e62af..5e3e226457 100644
--- a/src/interfaces/libpq/fe-connect.c
+++ b/src/interfaces/libpq/fe-connect.c
@@ -25,6 +25,7 @@
#include "common/ip.h"
#include "common/link-canary.h"
#include "common/scram-common.h"
+#include "common/zpq_stream.h"
#include "common/string.h"
#include "fe-auth.h"
#include "libpq-fe.h"
@@ -398,6 +399,10 @@ static const internalPQconninfoOption PQconninfoOptions[] = {
"Replication", "D", 5,
offsetof(struct pg_conn, replication)},
+ {"compression", "PGCOMPRESSION", "off", NULL,
+ "Libpq-compression", "", 16,
+ offsetof(struct pg_conn, compression)},
+
{"target_session_attrs", "PGTARGETSESSIONATTRS",
DefaultTargetSessionAttrs, NULL,
"Target-Session-Attrs", "", 15, /* sizeof("prefer-standby") = 15 */
@@ -523,6 +528,7 @@ pqDropConnection(PGconn *conn, bool flushInput)
{
io_stream_destroy(conn->io_stream);
conn->io_stream = NULL;
+ conn->zpqStream = NULL;
/* Close the socket itself */
if (conn->sock != PGINVALID_SOCKET)
closesocket(conn->sock);
@@ -1713,6 +1719,34 @@ connectOptions2(PGconn *conn)
goto oom_error;
}
+ /*
+ * validate compression option
+ */
+ if (conn->compression && conn->compression[0])
+ {
+ pg_compress_specification compressors[COMPRESSION_ALGORITHM_COUNT];
+ size_t n_compressors;
+ int rc = zpq_parse_compression_setting(conn->compression, compressors, &n_compressors);
+
+ if (rc == -1)
+ {
+ conn->status = CONNECTION_BAD;
+ appendPQExpBuffer(&conn->errorMessage,
+ libpq_gettext("invalid %s value: \"%s\"\n"),
+ "compression", conn->compression);
+ return false;
+ }
+
+ if (rc == 1 && n_compressors == 0)
+ {
+ conn->status = CONNECTION_BAD;
+ appendPQExpBuffer(&conn->errorMessage,
+ libpq_gettext("no supported algorithms found, %s value: \"%s\"\n"),
+ "compression", conn->compression);
+ return false;
+ }
+ }
+
/*
* validate target_session_attrs option, and set target_server_type
*/
@@ -3347,6 +3381,20 @@ keep_going: /* We will come back to here until there is
goto error_return;
}
+ /*
+ * By this point we have set up any encryption layers needed,
+ * so we can inject compression processing if requested.
+ */
+ if (!conn->zpqStream && conn->n_compressors > 0)
+ {
+ conn->zpqStream = zpq_create(conn->compressors, conn->n_compressors, conn->io_stream);
+ if (!conn->zpqStream)
+ {
+ libpq_append_conn_error(conn, "failed to set up compression stream");
+ goto error_return;
+ }
+ }
+
/*
* Send the startup packet.
*
@@ -3834,14 +3882,22 @@ keep_going: /* We will come back to here until there is
}
else if (beresp == PqMsg_NegotiateProtocolVersion)
{
- if (pqGetNegotiateProtocolVersion3(conn))
+ if ((res = pqProcessNegotiateProtocolVersion3(conn)) < 0)
{
libpq_append_conn_error(conn, "received invalid protocol negotiation message");
goto error_return;
}
/* OK, we read the message; mark data consumed */
conn->inStart = conn->inCursor;
- goto error_return;
+
+ if (res == 0)
+ {
+ goto error_return;
+ }
+ else
+ {
+ goto keep_going;
+ }
}
/* It is an authentication request. */
@@ -4270,6 +4326,50 @@ error_return:
return PGRES_POLLING_FAILED;
}
+/*
+ * Based on server GUC libpq_compression, establishes a zpq_stream to compres
+ * protocol traffic. Should only be called once per connection lifecycle, as the
+ * zpq_stream cannot be reconfigured without restarting the connection
+ */
+void
+pqConfigureCompression(PGconn *conn, const char *compressors_str)
+{
+ pg_compress_specification be_compressors[COMPRESSION_ALGORITHM_COUNT];
+ size_t n_be_compressors;
+
+ zpq_deserialize_compressors(compressors_str, be_compressors, &n_be_compressors);
+ if (conn->zpqStream)
+ {
+ pg_compress_algorithm algorithms[COMPRESSION_ALGORITHM_COUNT];
+ size_t n_algorithms = 0;
+
+ if (n_be_compressors == 0)
+ {
+ return;
+ }
+
+ /*
+ * Intersect client and server compressors to determine the final list
+ * of the supported compressors. O(N^2) is negligible because of a
+ * small number of the compression methods.
+ */
+ for (size_t i = 0; i < conn->n_compressors; i++)
+ {
+ for (size_t j = 0; j < n_be_compressors; j++)
+ {
+ if (conn->compressors[i].algorithm == be_compressors[j].algorithm && conn->compressors[i].compress && be_compressors[j].decompress)
+ {
+ algorithms[n_algorithms] = conn->compressors[i].algorithm;
+ n_algorithms += 1;
+ break;
+ }
+ }
+ }
+
+ zpq_enable_compression(conn->zpqStream, algorithms, n_algorithms);
+ }
+}
+
/*
* internal_ping
@@ -4480,6 +4580,7 @@ freePGconn(PGconn *conn)
free(conn->fbappname);
free(conn->dbName);
free(conn->replication);
+ free(conn->compression);
free(conn->pguser);
if (conn->pgpass)
{
@@ -7163,6 +7264,24 @@ PQuser(const PGconn *conn)
return conn->pguser;
}
+char *
+PQcompression(const PGconn *conn)
+{
+ if (!conn || !conn->zpqStream)
+ return NULL;
+
+ return zpq_algorithms(conn->zpqStream);
+}
+
+char *
+PQserverCompression(const PGconn *conn)
+{
+ if (!conn)
+ return NULL;
+
+ return conn->server_compression;
+}
+
char *
PQpass(const PGconn *conn)
{
@@ -8048,7 +8167,8 @@ retry_masked:
strlcat(msgbuf, "\n", sizeof(msgbuf));
conn->write_err_msg = strdup(msgbuf);
/* Now claim the write succeeded */
- n = len;
+ *bytes_written = n;
+ n = 0;
break;
default:
@@ -8063,7 +8183,8 @@ retry_masked:
strlcat(msgbuf, "\n", sizeof(msgbuf));
conn->write_err_msg = strdup(msgbuf);
/* Now claim the write succeeded */
- n = len;
+ *bytes_written = n;
+ n = 0;
break;
}
}
diff --git a/src/interfaces/libpq/fe-exec.c b/src/interfaces/libpq/fe-exec.c
index b9511df2c2..b6158d29c2 100644
--- a/src/interfaces/libpq/fe-exec.c
+++ b/src/interfaces/libpq/fe-exec.c
@@ -1185,6 +1185,15 @@ pqSaveParameterStatus(PGconn *conn, const char *name, const char *value)
{
conn->scram_sha_256_iterations = atoi(value);
}
+ else if (strcmp(name, "libpq_compression") == 0)
+ {
+ if (conn->server_compression)
+ {
+ free(conn->server_compression);
+ }
+ conn->server_compression = strdup(value);
+ pqConfigureCompression(conn, value);
+ }
}
@@ -3962,6 +3971,12 @@ pqPipelineFlush(PGconn *conn)
return 0;
}
+int
+PQreadPending(PGconn *conn)
+{
+ return pqReadPending(conn);
+}
+
/*
* PQfreemem - safely frees memory allocated
diff --git a/src/interfaces/libpq/fe-misc.c b/src/interfaces/libpq/fe-misc.c
index 6772f2876d..9aeeb123b7 100644
--- a/src/interfaces/libpq/fe-misc.c
+++ b/src/interfaces/libpq/fe-misc.c
@@ -51,6 +51,8 @@
#include "pg_config_paths.h"
#include "port/pg_bswap.h"
+#include <common/zpq_stream.h>
+
static int pqPutMsgBytes(const void *buf, size_t len, PGconn *conn);
static int pqSendSome(PGconn *conn, int len);
static int pqSocketCheck(PGconn *conn, int forRead, int forWrite,
@@ -618,6 +620,14 @@ retry3:
conn->inBufSize - conn->inEnd, false);
if (nread < 0)
{
+ if (nread == ZS_DECOMPRESS_ERROR)
+ {
+ printfPQExpBuffer(&conn->errorMessage,
+ libpq_gettext("decompress error: %s\n"),
+ zpq_decompress_error(conn->zpqStream));
+ return -1;
+ }
+
switch (SOCK_ERRNO)
{
case EINTR:
@@ -713,6 +723,14 @@ retry4:
conn->inBufSize - conn->inEnd, false);
if (nread < 0)
{
+ if (nread == ZS_DECOMPRESS_ERROR)
+ {
+ printfPQExpBuffer(&conn->errorMessage,
+ libpq_gettext("decompress error: %s\n"),
+ zpq_decompress_error(conn->zpqStream));
+ return -1;
+ }
+
switch (SOCK_ERRNO)
{
case EINTR:
@@ -822,7 +840,7 @@ pqSendSome(PGconn *conn, int len)
}
/* while there's still data to send */
- while (len > 0)
+ while (len > 0 || io_stream_buffered_write_data(conn->io_stream))
{
size_t sent;
int rc;
@@ -883,7 +901,7 @@ pqSendSome(PGconn *conn, int len)
}
}
- if (len > 0)
+ if (len > 0 || rc < 0 || io_stream_buffered_write_data(conn->io_stream))
{
/*
* We didn't send it all, wait till we can send more.
@@ -951,7 +969,7 @@ pqSendSome(PGconn *conn, int len)
int
pqFlush(PGconn *conn)
{
- if (conn->outCount > 0)
+ if (conn->outCount > 0 || io_stream_buffered_write_data(conn->io_stream))
{
if (conn->Pfdebug)
fflush(conn->Pfdebug);
@@ -976,6 +994,8 @@ pqFlush(PGconn *conn)
int
pqWait(int forRead, int forWrite, PGconn *conn)
{
+ if (forRead && conn->inCursor < conn->inEnd)
+ return 0;
return pqWaitTimed(forRead, forWrite, conn, (time_t) -1);
}
@@ -1030,8 +1050,9 @@ pqWriteReady(PGconn *conn)
* or both. Returns >0 if one or more conditions are met, 0 if it timed
* out, -1 if an error occurred.
*
- * If SSL is in use, the SSL buffer is checked prior to checking the socket
- * for read data directly.
+ * If protocol layers are using buffering, the buffers are checked prior
+ * to checking the socket for read data directly.
+ *
*/
static int
pqSocketCheck(PGconn *conn, int forRead, int forWrite, time_t end_time)
@@ -1068,6 +1089,22 @@ pqSocketCheck(PGconn *conn, int forRead, int forWrite, time_t end_time)
return result;
}
+/*
+ * Check if there is some data pending in stream buffers.
+ * Returns -1 on failure, 0 if no, 1 if yes.
+ */
+int
+pqReadPending(PGconn *conn)
+{
+ if (!conn)
+ return -1;
+
+ if (io_stream_buffered_read_data(conn->io_stream))
+ /* short-circuit the select */
+ return 1;
+
+ return 0;
+}
/*
* Check a file descriptor for read and/or write data, possibly waiting.
diff --git a/src/interfaces/libpq/fe-protocol3.c b/src/interfaces/libpq/fe-protocol3.c
index 8c4ec079ca..1b0dc47949 100644
--- a/src/interfaces/libpq/fe-protocol3.c
+++ b/src/interfaces/libpq/fe-protocol3.c
@@ -74,6 +74,20 @@ pqParseInput3(PGconn *conn)
*/
for (;;)
{
+ /*
+ * Read any available buffered data io_stream may be null when
+ * draining the final messages from an aborted connection
+ */
+ if (conn->io_stream && pqReadPending(conn) && (conn->inBufSize - conn->inEnd > 0))
+ {
+ int rc = io_stream_read(conn->io_stream, conn->inBuffer + conn->inEnd, conn->inBufSize - conn->inEnd, true);
+
+ if (rc > 0)
+ {
+ conn->inEnd += rc;
+ }
+ }
+
/*
* Try to read a message. First get the type code and length. Return
* if not enough data.
@@ -1404,16 +1418,18 @@ reportErrorPosition(PQExpBuffer msg, const char *query, int loc, int encoding)
/*
* Attempt to read a NegotiateProtocolVersion message.
* Entry: 'v' message type and length have already been consumed.
- * Exit: returns 0 if successfully consumed message.
+ * Exit: returns 0 if successfully consumed message and should exit,
+ * returns 1 if successfully consumed message and should continue with warning,
* returns EOF if not enough data.
*/
int
-pqGetNegotiateProtocolVersion3(PGconn *conn)
+pqProcessNegotiateProtocolVersion3(PGconn *conn)
{
int tmp;
ProtocolVersion their_version;
int num;
PQExpBufferData buf;
+ int retval = 1;
if (pqGetInt(&tmp, 4, conn) != 0)
return EOF;
@@ -1436,9 +1452,12 @@ pqGetNegotiateProtocolVersion3(PGconn *conn)
}
if (their_version < conn->pversion)
+ {
libpq_append_conn_error(conn, "protocol version not supported by server: client uses %u.%u, server supports up to %u.%u",
PG_PROTOCOL_MAJOR(conn->pversion), PG_PROTOCOL_MINOR(conn->pversion),
PG_PROTOCOL_MAJOR(their_version), PG_PROTOCOL_MINOR(their_version));
+ retval = 0;
+ }
if (num > 0)
{
appendPQExpBuffer(&conn->errorMessage,
@@ -1446,14 +1465,26 @@ pqGetNegotiateProtocolVersion3(PGconn *conn)
"protocol extensions not supported by server: %s", num),
buf.data);
appendPQExpBufferChar(&conn->errorMessage, '\n');
+
+ /*
+ * We can continue to use the connection if the server doesn't support
+ * compression
+ */
+ if (num > 1 || (strcmp(buf.data, "_pq_.libpq_compression") != 0))
+ {
+ retval = 0;
+ }
}
/* neither -- server shouldn't have sent it */
if (!(their_version < conn->pversion) && !(num > 0))
+ {
libpq_append_conn_error(conn, "invalid %s message", "NegotiateProtocolVersion");
+ retval = 0;
+ }
termPQExpBuffer(&buf);
- return 0;
+ return retval;
}
@@ -1764,7 +1795,7 @@ pqGetCopyData3(PGconn *conn, char **buffer, int async)
if (msgLength == 0)
{
/* Don't block if async read requested */
- if (async)
+ if (async && !pqReadPending(conn))
return 0;
/* Need to load more data */
if (pqWait(true, false, conn) ||
@@ -2246,6 +2277,46 @@ pqBuildStartupPacket3(PGconn *conn, int *packetlen,
return startpacket;
}
+/*
+ * Build semicolon-separated list of compression algorithms requested by client.
+ * It can be either explicitly specified by user in connection string, or
+ * include all algorithms supported by client library.
+ * This function returns true if the compression string is successfully parsed and
+ * stores a comma-separated list of algorithms in *client_compressors.
+ * If compression is disabled, then NULL is assigned to *client_compressors.
+ * Also it creates an array of compressor descriptors, each element of which corresponds to
+ * the corresponding algorithm name in *client_compressors list. This array is stored in PGconn
+ * and is used during handshake when a compression acknowledgment response is received from the server.
+ */
+static bool
+build_compressors_list(PGconn *conn, char **client_compressors, bool build_descriptors)
+{
+ pg_compress_specification compressors[COMPRESSION_ALGORITHM_COUNT];
+ size_t n_compressors;
+
+ if (zpq_parse_compression_setting(conn->compression, compressors, &n_compressors) < 0)
+ {
+ return false;
+ }
+
+ *client_compressors = NULL;
+ if (build_descriptors)
+ {
+ memcpy(conn->compressors, compressors, sizeof(compressors));
+ conn->n_compressors = n_compressors;
+ }
+
+ if (n_compressors == 0)
+ {
+ /* no compressors available, return */
+ return true;
+ }
+
+ *client_compressors = zpq_serialize_compressors(compressors, n_compressors);
+
+ return true;
+}
+
/*
* Build a startup packet given a filled-in PGconn structure.
*
@@ -2292,6 +2363,19 @@ build_startup_packet(const PGconn *conn, char *packet,
ADD_STARTUP_OPTION("replication", conn->replication);
if (conn->pgoptions && conn->pgoptions[0])
ADD_STARTUP_OPTION("options", conn->pgoptions);
+ if (conn->compression && conn->compression[0])
+ {
+ char *client_compression_algorithms;
+
+ if (build_compressors_list((PGconn *) conn, &client_compression_algorithms, packet == NULL))
+ {
+ if (client_compression_algorithms)
+ {
+ ADD_STARTUP_OPTION("_pq_.libpq_compression", client_compression_algorithms);
+ free(client_compression_algorithms);
+ }
+ }
+ }
if (conn->send_appname)
{
/* Use appname if present, otherwise use fallback */
diff --git a/src/interfaces/libpq/fe-secure-openssl.c b/src/interfaces/libpq/fe-secure-openssl.c
index b9706cb151..69a2c86c7f 100644
--- a/src/interfaces/libpq/fe-secure-openssl.c
+++ b/src/interfaces/libpq/fe-secure-openssl.c
@@ -90,7 +90,7 @@ IoStreamProcessor pgtls_processor = {
.read = (io_stream_read_func) pgtls_read,
.write = (io_stream_write_func) pgtls_write,
.buffered_read_data = (io_stream_predicate) pgtls_read_pending,
- .destroy = (io_stream_destroy_func) pgtls_close
+ .destroy = (io_stream_consumer) pgtls_close
};
static bool pq_init_ssl_lib = true;
diff --git a/src/interfaces/libpq/libpq-fe.h b/src/interfaces/libpq/libpq-fe.h
index 97762d56f5..fb7d8a9471 100644
--- a/src/interfaces/libpq/libpq-fe.h
+++ b/src/interfaces/libpq/libpq-fe.h
@@ -342,6 +342,8 @@ extern char *PQhostaddr(const PGconn *conn);
extern char *PQport(const PGconn *conn);
extern char *PQtty(const PGconn *conn);
extern char *PQoptions(const PGconn *conn);
+extern char *PQcompression(const PGconn *conn);
+extern char *PQserverCompression(const PGconn *conn);
extern ConnStatusType PQstatus(const PGconn *conn);
extern PGTransactionStatusType PQtransactionStatus(const PGconn *conn);
extern const char *PQparameterStatus(const PGconn *conn,
@@ -501,6 +503,8 @@ extern PGPing PQpingParams(const char *const *keywords,
/* Force the write buffer to be written (or at least try) */
extern int PQflush(PGconn *conn);
+extern int PQreadPending(PGconn *conn);
+
/*
* "Fast path" interface --- not really recommended for application
* use
diff --git a/src/interfaces/libpq/libpq-int.h b/src/interfaces/libpq/libpq-int.h
index 1314663213..4ebd12e420 100644
--- a/src/interfaces/libpq/libpq-int.h
+++ b/src/interfaces/libpq/libpq-int.h
@@ -40,6 +40,7 @@
/* include stuff common to fe and be */
#include "libpq/pqcomm.h"
+#include "common/zpq_stream.h"
/* include stuff found in fe only */
#include "fe-auth-sasl.h"
#include "pqexpbuffer.h"
@@ -406,6 +407,16 @@ struct pg_conn
char *gssdelegation; /* Try to delegate GSS credentials? (0 or 1) */
char *ssl_min_protocol_version; /* minimum TLS protocol version */
char *ssl_max_protocol_version; /* maximum TLS protocol version */
+ char *compression; /* stream compression (boolean value, "any" or
+ * list of compression algorithms separated by
+ * comma) */
+ pg_compress_specification compressors[COMPRESSION_ALGORITHM_COUNT]; /* descriptors of
+ * compression
+ * algorithms chosen by
+ * client */
+ unsigned n_compressors; /* size of compressors array */
+ char *server_compression; /* compression settings set by server */
+
char *target_session_attrs; /* desired session properties */
char *require_auth; /* name of the expected auth method */
char *load_balance_hosts; /* load balance over hosts */
@@ -621,6 +632,9 @@ struct pg_conn
/* Buffer for receiving various parts of messages */
PQExpBufferData workBuffer; /* expansible string */
+
+ /* Compression stream */
+ ZpqStream *zpqStream;
};
/* PGcancel stores all data necessary to cancel a connection. A copy of this
@@ -680,6 +694,7 @@ extern void pqDropConnection(PGconn *conn, bool flushInput);
extern int pqPacketSend(PGconn *conn, char pack_type,
const void *buf, size_t buf_len);
extern bool pqGetHomeDirectory(char *buf, int bufsize);
+extern void pqConfigureCompression(PGconn *conn, const char *compressors);
extern pgthreadlock_t pg_g_threadlock;
@@ -712,7 +727,7 @@ extern void pqParseInput3(PGconn *conn);
extern int pqGetErrorNotice3(PGconn *conn, bool isError);
extern void pqBuildErrorMessage3(PQExpBuffer msg, const PGresult *res,
PGVerbosity verbosity, PGContextVisibility show_context);
-extern int pqGetNegotiateProtocolVersion3(PGconn *conn);
+extern int pqProcessNegotiateProtocolVersion3(PGconn *conn);
extern int pqGetCopyData3(PGconn *conn, char **buffer, int async);
extern int pqGetline3(PGconn *conn, char *s, int maxlen);
extern int pqGetlineAsync3(PGconn *conn, char *buffer, int bufsize);
@@ -750,6 +765,7 @@ extern int pqWaitTimed(int forRead, int forWrite, PGconn *conn,
time_t finish_time);
extern int pqReadReady(PGconn *conn);
extern int pqWriteReady(PGconn *conn);
+extern int pqReadPending(PGconn *conn);
/* === in fe-secure.c === */
--
2.42.0
[application/octet-stream] v3-0003-Add-network-traffic-stats.patch (22.7K, ../../CACzsqT5-7xfbz+Si35TBYHzerNX3XJVzAUH9AewQ+Pp13fYBoQ@mail.gmail.com/4-v3-0003-Add-network-traffic-stats.patch)
download | inline diff:
From 7d3a7ffca211d36c43132280e1c775fb0459572e Mon Sep 17 00:00:00 2001
From: Jacob Burroughs <[email protected]>
Date: Fri, 15 Dec 2023 13:08:55 -0600
Subject: [PATCH v3 3/5] Add network traffic stats
Adds pg_stat_network_traffic to monitor both bytes sent over
the wire and "logical" protocol bytes, allowing for measurement
of compression ratio (and SSL overhead)
---
doc/src/sgml/monitoring.sgml | 100 ++++++++++++++++++++
src/backend/catalog/system_views.sql | 10 ++
src/backend/libpq/pqcomm.c | 8 ++
src/backend/utils/activity/backend_status.c | 86 +++++++++++++++++
src/backend/utils/adt/pgstatfuncs.c | 50 +++++++++-
src/include/catalog/pg_proc.dat | 14 ++-
src/include/utils/backend_status.h | 10 ++
src/test/regress/expected/rules.out | 15 ++-
8 files changed, 285 insertions(+), 8 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index b804eb8b5e..8f4a55d9b2 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -350,6 +350,15 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser
</entry>
</row>
+ <row>
+ <entry><structname>pg_stat_network_traffic</structname><indexterm><primary>pg_stat_network_traffic</primary></indexterm></entry>
+ <entry>One row per connection (regular and replication), showing information about
+ network traffic of this connection.
+ See <link linkend="monitoring-pg-stat-network-traffic-view">
+ <structname>pg_stat_network_traffic</structname></link> for details.
+ </entry>
+ </row>
+
<row>
<entry><structname>pg_stat_ssl</structname><indexterm><primary>pg_stat_ssl</primary></indexterm></entry>
<entry>One row per connection (regular and replication), showing information about
@@ -2176,6 +2185,97 @@ description | Waiting for a newly initialized WAL file to reach durable storage
</sect2>
+ <sect2 id="monitoring-pg-stat-network-traffic-view">
+ <title><structname>pg_stat_network_traffic</structname></title>
+
+ <indexterm>
+ <primary>pg_stat_network_traffic</primary>
+ </indexterm>
+
+ <para>
+ The <structname>pg_stat_network_traffic</structname> view will contain one row per
+ backend or WAL sender process, showing statistics about network traffic on
+ this connection. It can be joined to <structname>pg_stat_activity</structname>
+ or <structname>pg_stat_replication</structname> on the
+ <structfield>pid</structfield> column to get more details about the
+ connection.
+ </para>
+
+ <table id="pg-stat-network-traffic-view" xreflabel="pg_stat_network_traffic">
+ <title><structname>pg_stat_network_traffic</structname> View</title>
+ <tgroup cols="1">
+ <thead>
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ Column Type
+ </para>
+ <para>
+ Description
+ </para></entry>
+ </row>
+ </thead>
+
+ <tbody>
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>pid</structfield> <type>integer</type>
+ </para>
+ <para>
+ Process ID of a backend or WAL sender process
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>rx_socket_bytes</structfield> <type>integer</type>
+ </para>
+ <para>
+ Number of bytes received from the underlying socket of this connection
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>tx_socket_bytes</structfield> <type>integer</type>
+ </para>
+ <para>
+ Number of bytes sent to the underlying socket of this connection
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>rx_pq_bytes</structfield> <type>integer</type>
+ </para>
+ <para>
+ Number of bytes of raw libpq traffic recieved from this connection.
+ This can be used to estimate the overhead of SSL, which will generally
+ make this number smaller than <structfield>rx_socket_bytes</structfield>,
+ and to estimate the compression ratio when using protocol compression,
+ as compression will generally make this number larger than
+ <structfield>rx_socket_bytes</structfield>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>tx_pq_bytes</structfield> <type>integer</type>
+ </para>
+ <para>
+ Number of bytes of raw libpq traffic sent through this connection.
+ This can be used to estimate the overhead of SSL, which will generally
+ make this number smaller than <structfield>tx_socket_bytes</structfield>,
+ and to estimate the compression ratio when using protocol compression,
+ as compression will generally make this number larger than
+ <structfield>tx_socket_bytes</structfield>.
+ </para></entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ </sect2>
+
<sect2 id="monitoring-pg-stat-ssl-view">
<title><structname>pg_stat_ssl</structname></title>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 058fc47c91..2cd4d61305 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -893,6 +893,16 @@ CREATE VIEW pg_stat_activity AS
LEFT JOIN pg_database AS D ON (S.datid = D.oid)
LEFT JOIN pg_authid AS U ON (S.usesysid = U.oid);
+CREATE VIEW pg_stat_network_traffic AS
+ SELECT
+ S.pid,
+ S.rx_socket_bytes,
+ S.tx_socket_bytes,
+ S.rx_pq_bytes,
+ S.tx_pq_bytes
+ FROM pg_stat_get_activity(NULL) AS S
+ WHERE S.client_port IS NOT NULL;
+
CREATE VIEW pg_stat_replication AS
SELECT
S.pid,
diff --git a/src/backend/libpq/pqcomm.c b/src/backend/libpq/pqcomm.c
index ed4aad57e6..c44840881c 100644
--- a/src/backend/libpq/pqcomm.c
+++ b/src/backend/libpq/pqcomm.c
@@ -966,6 +966,8 @@ socket_read(IoStreamLayer * self, Port *port, void *ptr, size_t len, bool buffer
#ifdef WIN32
pgwin32_noblock = false;
#endif
+ if (n > 0)
+ pgstat_report_rx_socket_traffic(n);
return n;
}
@@ -982,6 +984,8 @@ socket_write(IoStreamLayer * self, Port *port, const void *ptr, size_t len, size
#ifdef WIN32
pgwin32_noblock = false;
#endif
+ if (n > 0)
+ pgstat_report_tx_socket_traffic(n);
if (n >= 0)
{
@@ -1009,6 +1013,8 @@ io_read_with_wait(Port *port, void *ptr, size_t len)
retry:
port->waitfor = WL_SOCKET_READABLE;
n = io_stream_read(port->io_stream, ptr, len, false);
+ if (n > 0)
+ pgstat_report_rx_pq_traffic(n);
/* In blocking mode, wait until the socket is ready */
if (n < 0 && !port->noblock && (errno == EWOULDBLOCK || errno == EAGAIN))
@@ -1092,6 +1098,8 @@ retry:
*/
rc = io_stream_write(port->io_stream, ptr + *bytes_processed, len - *bytes_processed, &count);
*bytes_processed += count;
+ if (count > 0)
+ pgstat_report_tx_pq_traffic(count);
if (rc < 0 && !port->noblock && (errno == EWOULDBLOCK || errno == EAGAIN))
{
diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c
index 6e734c6caf..c5712d86b8 100644
--- a/src/backend/utils/activity/backend_status.c
+++ b/src/backend/utils/activity/backend_status.c
@@ -338,6 +338,9 @@ pgstat_bestart(void)
lbeentry.st_xact_start_timestamp = 0;
lbeentry.st_databaseid = MyDatabaseId;
+ lbeentry.st_rx_socket_bytes = lbeentry.st_tx_socket_bytes =
+ lbeentry.st_rx_pq_bytes = lbeentry.st_tx_pq_bytes = 0;
+
/* We have userid for client-backends, wal-sender and bgworker processes */
if (lbeentry.st_backendType == B_BACKEND
|| lbeentry.st_backendType == B_WAL_SENDER
@@ -717,6 +720,89 @@ pgstat_report_xact_timestamp(TimestampTz tstamp)
PGSTAT_END_WRITE_ACTIVITY(beentry);
}
+/*
+ * Report network raw or compressed tx/rx traffic as the specified values.
+ */
+void
+pgstat_report_rx_socket_traffic(uint64 bytes)
+{
+ volatile PgBackendStatus *beentry = MyBEEntry;
+
+ if (!pgstat_track_activities || !beentry)
+ return;
+
+ /*
+ * Update my status entry, following the protocol of bumping
+ * st_changecount before and after. We use a volatile pointer here to
+ * ensure the compiler doesn't try to get cute.
+ */
+ PGSTAT_BEGIN_WRITE_ACTIVITY(beentry);
+
+ beentry->st_rx_socket_bytes += bytes;
+
+ PGSTAT_END_WRITE_ACTIVITY(beentry);
+}
+
+void
+pgstat_report_tx_socket_traffic(uint64 bytes)
+{
+ volatile PgBackendStatus *beentry = MyBEEntry;
+
+ if (!pgstat_track_activities || !beentry)
+ return;
+
+ /*
+ * Update my status entry, following the protocol of bumping
+ * st_changecount before and after. We use a volatile pointer here to
+ * ensure the compiler doesn't try to get cute.
+ */
+ PGSTAT_BEGIN_WRITE_ACTIVITY(beentry);
+
+ beentry->st_tx_socket_bytes += bytes;
+
+ PGSTAT_END_WRITE_ACTIVITY(beentry);
+}
+
+void
+pgstat_report_rx_pq_traffic(uint64 bytes)
+{
+ volatile PgBackendStatus *beentry = MyBEEntry;
+
+ if (!pgstat_track_activities || !beentry)
+ return;
+
+ /*
+ * Update my status entry, following the protocol of bumping
+ * st_changecount before and after. We use a volatile pointer here to
+ * ensure the compiler doesn't try to get cute.
+ */
+ PGSTAT_BEGIN_WRITE_ACTIVITY(beentry);
+
+ beentry->st_rx_pq_bytes += bytes;
+
+ PGSTAT_END_WRITE_ACTIVITY(beentry);
+}
+
+void
+pgstat_report_tx_pq_traffic(uint64 bytes)
+{
+ volatile PgBackendStatus *beentry = MyBEEntry;
+
+ if (!pgstat_track_activities || !beentry)
+ return;
+
+ /*
+ * Update my status entry, following the protocol of bumping
+ * st_changecount before and after. We use a volatile pointer here to
+ * ensure the compiler doesn't try to get cute.
+ */
+ PGSTAT_BEGIN_WRITE_ACTIVITY(beentry);
+
+ beentry->st_tx_pq_bytes += bytes;
+
+ PGSTAT_END_WRITE_ACTIVITY(beentry);
+}
+
/* ----------
* pgstat_read_current_status() -
*
diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c
index e65cbf41e9..1273944fd8 100644
--- a/src/backend/utils/adt/pgstatfuncs.c
+++ b/src/backend/utils/adt/pgstatfuncs.c
@@ -304,7 +304,7 @@ pg_stat_get_progress_info(PG_FUNCTION_ARGS)
Datum
pg_stat_get_activity(PG_FUNCTION_ARGS)
{
-#define PG_STAT_GET_ACTIVITY_COLS 31
+#define PG_STAT_GET_ACTIVITY_COLS 35
int num_backends = pgstat_fetch_stat_numbackends();
int curr_backend;
int pid = PG_ARGISNULL(0) ? -1 : PG_GETARG_INT32(0);
@@ -617,6 +617,10 @@ pg_stat_get_activity(PG_FUNCTION_ARGS)
nulls[30] = true;
else
values[30] = UInt64GetDatum(beentry->st_query_id);
+ values[31] = UInt64GetDatum(beentry->st_rx_socket_bytes);
+ values[32] = UInt64GetDatum(beentry->st_tx_socket_bytes);
+ values[33] = UInt64GetDatum(beentry->st_rx_pq_bytes);
+ values[34] = UInt64GetDatum(beentry->st_tx_pq_bytes);
}
else
{
@@ -646,6 +650,10 @@ pg_stat_get_activity(PG_FUNCTION_ARGS)
nulls[28] = true;
nulls[29] = true;
nulls[30] = true;
+ nulls[31] = true;
+ nulls[32] = true;
+ nulls[33] = true;
+ nulls[34] = true;
}
tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls);
@@ -876,6 +884,46 @@ pg_stat_get_backend_start(PG_FUNCTION_ARGS)
PG_RETURN_TIMESTAMPTZ(result);
}
+Datum
+pg_stat_get_network_traffic(PG_FUNCTION_ARGS)
+{
+#define PG_STAT_NETWORK_TRAFFIC_COLS 4
+ TupleDesc tupdesc;
+ Datum values[PG_STAT_NETWORK_TRAFFIC_COLS];
+ bool nulls[PG_STAT_NETWORK_TRAFFIC_COLS];
+ int32 beid = PG_GETARG_INT32(0);
+ PgBackendStatus *beentry;
+
+ if ((beentry = pgstat_get_beentry_by_backend_id(beid)) == NULL)
+ PG_RETURN_NULL();
+ else if (!HAS_PGSTAT_PERMISSIONS(beentry->st_userid))
+ PG_RETURN_NULL();
+
+ /* Initialise values and NULL flags arrays */
+ MemSet(values, 0, sizeof(values));
+ MemSet(nulls, 0, sizeof(nulls));
+
+ /* Initialise attributes information in the tuple descriptor */
+ tupdesc = CreateTemplateTupleDesc(PG_STAT_NETWORK_TRAFFIC_COLS);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 1, "rx_socket_bytes",
+ INT8OID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 2, "tx_socket_bytes",
+ INT8OID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 3, "rx_pq_bytes",
+ INT8OID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 4, "tx_pq_bytes",
+ INT8OID, -1, 0);
+ BlessTupleDesc(tupdesc);
+
+ /* Fill values and NULLs */
+ values[0] = UInt64GetDatum(beentry->st_rx_socket_bytes);
+ values[1] = UInt64GetDatum(beentry->st_tx_socket_bytes);
+ values[2] = UInt64GetDatum(beentry->st_rx_pq_bytes);
+ values[3] = UInt64GetDatum(beentry->st_tx_pq_bytes);
+
+ /* Returns the record as Datum */
+ PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls)));
+}
Datum
pg_stat_get_backend_client_addr(PG_FUNCTION_ARGS)
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 9052f5262a..cf108da019 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -5440,9 +5440,9 @@
proname => 'pg_stat_get_activity', prorows => '100', proisstrict => 'f',
proretset => 't', provolatile => 's', proparallel => 'r',
prorettype => 'record', proargtypes => 'int4',
- proallargtypes => '{int4,oid,int4,oid,text,text,text,text,text,timestamptz,timestamptz,timestamptz,timestamptz,inet,text,int4,xid,xid,text,bool,text,text,int4,text,numeric,text,bool,text,bool,bool,int4,int8}',
- proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}',
- proargnames => '{pid,datid,pid,usesysid,application_name,state,query,wait_event_type,wait_event,xact_start,query_start,backend_start,state_change,client_addr,client_hostname,client_port,backend_xid,backend_xmin,backend_type,ssl,sslversion,sslcipher,sslbits,ssl_client_dn,ssl_client_serial,ssl_issuer_dn,gss_auth,gss_princ,gss_enc,gss_delegation,leader_pid,query_id}',
+ proallargtypes => '{int4,oid,int4,oid,text,text,text,text,text,timestamptz,timestamptz,timestamptz,timestamptz,inet,text,int4,xid,xid,text,bool,text,text,int4,text,numeric,text,bool,text,bool,bool,int4,int8,int8,int8,int8,int8}',
+ proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}',
+ proargnames => '{pid,datid,pid,usesysid,application_name,state,query,wait_event_type,wait_event,xact_start,query_start,backend_start,state_change,client_addr,client_hostname,client_port,backend_xid,backend_xmin,backend_type,ssl,sslversion,sslcipher,sslbits,ssl_client_dn,ssl_client_serial,ssl_issuer_dn,gss_auth,gss_princ,gss_enc,gss_delegation,leader_pid,query_id,rx_socket_bytes,tx_socket_bytes,rx_pq_bytes,tx_pq_bytes}',
prosrc => 'pg_stat_get_activity' },
{ oid => '8403', descr => 'describe wait events',
proname => 'pg_get_wait_events', procost => '10', prorows => '250',
@@ -5797,6 +5797,14 @@
proargnames => '{stats_reset,prefetch,hit,skip_init,skip_new,skip_fpw,skip_rep,wal_distance,block_distance,io_depth}',
prosrc => 'pg_stat_get_recovery_prefetch' },
+{ oid => '9598', descr => 'statistics: information about network traffic',
+ proname => 'pg_stat_get_network_traffic', proisstrict => 'f', provolatile => 's',
+ proparallel => 'r', prorettype => 'record', proargtypes => 'int4',
+ proallargtypes => '{int4,int8,int8,int8,int8}',
+ proargmodes => '{i,o,o,o,o}',
+ proargnames => '{_beid,rx_socket_bytes,tx_socket_bytes,rx_pq_bytes,tx_pq_bytes}',
+ prosrc => 'pg_stat_get_network_traffic' },
+
{ oid => '2306', descr => 'statistics: information about SLRU caches',
proname => 'pg_stat_get_slru', prorows => '100', proisstrict => 'f',
proretset => 't', provolatile => 's', proparallel => 'r',
diff --git a/src/include/utils/backend_status.h b/src/include/utils/backend_status.h
index 75fc18c432..5df7bbd797 100644
--- a/src/include/utils/backend_status.h
+++ b/src/include/utils/backend_status.h
@@ -146,6 +146,12 @@ typedef struct PgBackendStatus
/* application name; MUST be null-terminated */
char *st_appname;
+ /* client-server traffic information */
+ uint64 st_rx_socket_bytes;
+ uint64 st_tx_socket_bytes;
+ uint64 st_rx_pq_bytes;
+ uint64 st_tx_pq_bytes;
+
/*
* Current command string; MUST be null-terminated. Note that this string
* possibly is truncated in the middle of a multi-byte character. As
@@ -321,6 +327,10 @@ extern void pgstat_report_query_id(uint64 query_id, bool force);
extern void pgstat_report_tempfile(size_t filesize);
extern void pgstat_report_appname(const char *appname);
extern void pgstat_report_xact_timestamp(TimestampTz tstamp);
+extern void pgstat_report_rx_socket_traffic(uint64 bytes);
+extern void pgstat_report_tx_socket_traffic(uint64 bytes);
+extern void pgstat_report_rx_pq_traffic(uint64 bytes);
+extern void pgstat_report_tx_pq_traffic(uint64 bytes);
extern const char *pgstat_get_backend_current_activity(int pid, bool checkUser);
extern const char *pgstat_get_crashed_backend_activity(int pid, char *buffer,
int buflen);
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index f645e8486b..33e1017109 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1760,7 +1760,7 @@ pg_stat_activity| SELECT s.datid,
s.query_id,
s.query,
s.backend_type
- FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, gss_delegation, leader_pid, query_id)
+ FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, gss_delegation, leader_pid, query_id, rx_socket_bytes, tx_socket_bytes, rx_pq_bytes, tx_pq_bytes)
LEFT JOIN pg_database d ON ((s.datid = d.oid)))
LEFT JOIN pg_authid u ON ((s.usesysid = u.oid)));
pg_stat_all_indexes| SELECT c.oid AS relid,
@@ -1880,7 +1880,7 @@ pg_stat_gssapi| SELECT pid,
gss_princ AS principal,
gss_enc AS encrypted,
gss_delegation AS credentials_delegated
- FROM pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, gss_delegation, leader_pid, query_id)
+ FROM pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, gss_delegation, leader_pid, query_id, rx_socket_bytes, tx_socket_bytes, rx_pq_bytes, tx_pq_bytes)
WHERE (client_port IS NOT NULL);
pg_stat_io| SELECT backend_type,
object,
@@ -1901,6 +1901,13 @@ pg_stat_io| SELECT backend_type,
fsync_time,
stats_reset
FROM pg_stat_get_io() b(backend_type, object, context, reads, read_time, writes, write_time, writebacks, writeback_time, extends, extend_time, op_bytes, hits, evictions, reuses, fsyncs, fsync_time, stats_reset);
+pg_stat_network_traffic| SELECT pid,
+ rx_socket_bytes,
+ tx_socket_bytes,
+ rx_pq_bytes,
+ tx_pq_bytes
+ FROM pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, gss_delegation, leader_pid, query_id, rx_socket_bytes, tx_socket_bytes, rx_pq_bytes, tx_pq_bytes)
+ WHERE (client_port IS NOT NULL);
pg_stat_progress_analyze| SELECT s.pid,
s.datid,
d.datname,
@@ -2082,7 +2089,7 @@ pg_stat_replication| SELECT s.pid,
w.sync_priority,
w.sync_state,
w.reply_time
- FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, gss_delegation, leader_pid, query_id)
+ FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, gss_delegation, leader_pid, query_id, rx_socket_bytes, tx_socket_bytes, rx_pq_bytes, tx_pq_bytes)
JOIN pg_stat_get_wal_senders() w(pid, state, sent_lsn, write_lsn, flush_lsn, replay_lsn, write_lag, flush_lag, replay_lag, sync_priority, sync_state, reply_time) ON ((s.pid = w.pid)))
LEFT JOIN pg_authid u ON ((s.usesysid = u.oid)));
pg_stat_replication_slots| SELECT s.slot_name,
@@ -2116,7 +2123,7 @@ pg_stat_ssl| SELECT pid,
ssl_client_dn AS client_dn,
ssl_client_serial AS client_serial,
ssl_issuer_dn AS issuer_dn
- FROM pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, gss_delegation, leader_pid, query_id)
+ FROM pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, gss_delegation, leader_pid, query_id, rx_socket_bytes, tx_socket_bytes, rx_pq_bytes, tx_pq_bytes)
WHERE (client_port IS NOT NULL);
pg_stat_subscription| SELECT su.oid AS subid,
su.subname,
--
2.42.0
[application/octet-stream] v3-0004-Add-basic-test-for-compression-functionality.patch (6.9K, ../../CACzsqT5-7xfbz+Si35TBYHzerNX3XJVzAUH9AewQ+Pp13fYBoQ@mail.gmail.com/5-v3-0004-Add-basic-test-for-compression-functionality.patch)
download | inline diff:
From 73d58dfb71c14109402d396d0a388135be42bb6c Mon Sep 17 00:00:00 2001
From: Jacob Burroughs <[email protected]>
Date: Mon, 18 Dec 2023 14:15:18 -0600
Subject: [PATCH v3 4/5] Add basic test for compression functionality
---
src/Makefile.global.in | 2 +
src/interfaces/libpq/Makefile | 7 +-
src/interfaces/libpq/meson.build | 8 +-
src/interfaces/libpq/t/005_compression.pl | 94 +++++++++++++++++++++++
4 files changed, 108 insertions(+), 3 deletions(-)
create mode 100644 src/interfaces/libpq/t/005_compression.pl
diff --git a/src/Makefile.global.in b/src/Makefile.global.in
index f8e461cbad..e8fad07936 100644
--- a/src/Makefile.global.in
+++ b/src/Makefile.global.in
@@ -195,9 +195,11 @@ with_ldap = @with_ldap@
with_libxml = @with_libxml@
with_libxslt = @with_libxslt@
with_llvm = @with_llvm@
+with_lz4 = @with_lz4@
with_system_tzdata = @with_system_tzdata@
with_uuid = @with_uuid@
with_zlib = @with_zlib@
+with_zstd = @with_zstd@
enable_rpath = @enable_rpath@
enable_nls = @enable_nls@
enable_debug = @enable_debug@
diff --git a/src/interfaces/libpq/Makefile b/src/interfaces/libpq/Makefile
index df8a1f31b2..12dc5868b6 100644
--- a/src/interfaces/libpq/Makefile
+++ b/src/interfaces/libpq/Makefile
@@ -14,6 +14,9 @@ top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
export with_ssl
+export with_zlib
+export with_lz4
+export with_zstd
PGFILEDESC = "PostgreSQL Access Library"
@@ -78,9 +81,9 @@ endif
# that are built correctly for use in a shlib.
SHLIB_LINK_INTERNAL = -lpgcommon_shlib -lpgport_shlib
ifneq ($(PORTNAME), win32)
-SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi_krb5 -lgss -lgssapi -lssl -lsocket -lnsl -lresolv -lintl -lm, $(LIBS)) $(LDAP_LIBS_FE) $(PTHREAD_LIBS)
+SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi_krb5 -lgss -lgssapi -lssl -lsocket -lnsl -lresolv -lintl -lm -lz -llz4 -lzstd, $(LIBS)) $(LDAP_LIBS_FE) $(PTHREAD_LIBS)
else
-SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi32 -lssl -lsocket -lnsl -lresolv -lintl -lm $(PTHREAD_LIBS), $(LIBS)) $(LDAP_LIBS_FE)
+SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi32 -lssl -lsocket -lnsl -lresolv -lintl -lm -lz -llz4 -lzstd $(PTHREAD_LIBS), $(LIBS)) $(LDAP_LIBS_FE)
endif
ifeq ($(PORTNAME), win32)
SHLIB_LINK += -lshell32 -lws2_32 -lsecur32 $(filter -leay32 -lssleay32 -lcomerr32 -lkrb5_32, $(LIBS))
diff --git a/src/interfaces/libpq/meson.build b/src/interfaces/libpq/meson.build
index 80e6a15adf..e9b3c22a52 100644
--- a/src/interfaces/libpq/meson.build
+++ b/src/interfaces/libpq/meson.build
@@ -118,8 +118,14 @@ tests += {
't/002_api.pl',
't/003_load_balance_host_list.pl',
't/004_load_balance_dns.pl',
+ 't/005_compression.pl',
],
- 'env': {'with_ssl': ssl_library},
+ 'env': {
+ 'with_ssl': ssl_library,
+ 'with_zlib': zlib.found() ? 'yes' : 'no',
+ 'with_lz4': lz4.found() ? 'yes' : 'no',
+ 'with_zstd': zstd.found() ? 'yes' : 'no',
+ },
},
}
diff --git a/src/interfaces/libpq/t/005_compression.pl b/src/interfaces/libpq/t/005_compression.pl
new file mode 100644
index 0000000000..95c8575d80
--- /dev/null
+++ b/src/interfaces/libpq/t/005_compression.pl
@@ -0,0 +1,94 @@
+# Copyright (c) 2023, PostgreSQL Global Development Group
+use strict;
+use warnings FATAL => 'all';
+use Config;
+use PostgreSQL::Test::Utils;
+use PostgreSQL::Test::Cluster;
+use Test::More;
+
+my $withZlib = $ENV{with_zlib} eq 'yes';
+my $withLz4 = $ENV{with_lz4} eq 'yes';
+my $withZstd = $ENV{with_zstd} eq 'yes';
+if (!$withZlib && !$withLz4 && !$withZstd)
+{
+ plan skip_all => 'no compression methods available';
+}
+
+my $node = PostgreSQL::Test::Cluster->new('primary');
+$node->init;
+$node->append_conf('postgresql.conf', "libpq_compression = off");
+$node->start;
+
+# Use a string that any reasonable compression algorithm will compress
+my $compressableString = "a" x 1000;
+
+my $result;
+$result = $node->safe_psql("postgres",
+ "SELECT '$compressableString'; SELECT rx_socket_bytes < rx_pq_bytes, tx_socket_bytes < tx_pq_bytes FROM pg_stat_network_traffic;",
+ connstr => $node->connstr . " compression=on");
+is( (split "\n", $result)[-1], "f|f", 'successfully does not compress if server-side compression is disabled');
+
+$node->append_conf('postgresql.conf', "libpq_compression = on");
+$node->reload;
+
+$result = $node->safe_psql("postgres",
+ "SELECT '$compressableString'; SELECT rx_socket_bytes < rx_pq_bytes, tx_socket_bytes < tx_pq_bytes FROM pg_stat_network_traffic;",
+ connstr => $node->connstr . " compression=on");
+is( (split "\n", $result)[-1], "t|t", 'successfully compresses bidirectionally with default algorithms');
+
+my $test_algorithm;
+if ($withZlib)
+{
+ $test_algorithm = "gzip";
+}
+elsif($withLz4)
+{
+ $test_algorithm = "lz4";
+}
+elsif($withZstd)
+{
+ $test_algorithm = "zstd";
+}
+$result = $node->safe_psql("postgres",
+ "SELECT '$compressableString'; SELECT rx_socket_bytes < rx_pq_bytes, tx_socket_bytes < tx_pq_bytes FROM pg_stat_network_traffic;",
+ connstr => $node->connstr . " compression=$test_algorithm:compress=off");
+is( (split "\n", $result)[-1], "f|t", 'successfully compresses unidirectionally with client compression disabled');
+
+$node->append_conf('postgresql.conf', "libpq_compression = '$test_algorithm:compress=off'");
+$node->reload;
+$result = $node->safe_psql("postgres",
+ "SELECT '$compressableString'; SELECT rx_socket_bytes < rx_pq_bytes, tx_socket_bytes < tx_pq_bytes FROM pg_stat_network_traffic;",
+ connstr => "compression=on");
+is( (split "\n", $result)[-1], "t|f", 'successfully compresses unidirectionally with server compression algorithm');
+
+$node->append_conf('postgresql.conf', 'libpq_compression = on');
+$node->reload;
+
+SKIP: {
+ skip "gzip not available", 1 unless $ENV{with_zlib};
+
+ $result = $node->safe_psql("postgres",
+ "SELECT '$compressableString'; SELECT rx_socket_bytes < rx_pq_bytes, tx_socket_bytes < tx_pq_bytes FROM pg_stat_network_traffic;",
+ connstr => $node->connstr . " compression=gzip");
+ is( (split "\n", $result)[-1], "t|t", 'successfully compresses bidirectionally with gzip');
+}
+
+SKIP: {
+ skip "lz4 not available", 1 unless $ENV{with_lz4};
+
+ $result = $node->safe_psql("postgres",
+ "SELECT '$compressableString'; SELECT rx_socket_bytes < rx_pq_bytes, tx_socket_bytes < tx_pq_bytes FROM pg_stat_network_traffic;",
+ connstr => $node->connstr . " compression=lz4");
+ is( (split "\n", $result)[-1], "t|t", 'successfully compresses bidirectionally with lz4');
+}
+
+SKIP: {
+ skip "zstd not available", 1 unless $ENV{with_zstd};
+
+ $result = $node->safe_psql("postgres",
+ "SELECT '$compressableString'; SELECT rx_socket_bytes < rx_pq_bytes, tx_socket_bytes < tx_pq_bytes FROM pg_stat_network_traffic;",
+ connstr => $node->connstr . " compression=zstd");
+ is( (split "\n", $result)[-1], "t|t", 'successfully compresses bidirectionally with zstd');
+}
+
+done_testing();
--
2.42.0
[application/octet-stream] v3-0005-DO-NOT-MERGE-enable-compression-for-CI.patch (3.7K, ../../CACzsqT5-7xfbz+Si35TBYHzerNX3XJVzAUH9AewQ+Pp13fYBoQ@mail.gmail.com/6-v3-0005-DO-NOT-MERGE-enable-compression-for-CI.patch)
download | inline diff:
From 3b48d3ae99682b70fa8ad169faa628d983e2a04d Mon Sep 17 00:00:00 2001
From: Jacob Burroughs <[email protected]>
Date: Sun, 17 Dec 2023 16:09:23 -0600
Subject: [PATCH v3 5/5] DO NOT MERGE: enable compression for CI
---
src/backend/libpq/compression.c | 2 +-
src/backend/utils/misc/guc_tables.c | 2 +-
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/common/zpq_stream.c | 3 ++-
src/interfaces/libpq/fe-connect.c | 4 +++-
5 files changed, 8 insertions(+), 5 deletions(-)
diff --git a/src/backend/libpq/compression.c b/src/backend/libpq/compression.c
index bd0f97b32a..cd43c6dbe4 100644
--- a/src/backend/libpq/compression.c
+++ b/src/backend/libpq/compression.c
@@ -17,7 +17,7 @@
#include "utils/guc_hooks.h"
/* GUC variable containing the allowed compression algorithms list (separated by semicolon) */
-char *libpq_compress_algorithms = "off";
+char *libpq_compress_algorithms = "on";
pg_compress_specification libpq_compressors[COMPRESSION_ALGORITHM_COUNT];
size_t libpq_n_compressors = 0;
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index a98f2ccd97..ee7d993a5d 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -4550,7 +4550,7 @@ struct config_string ConfigureNamesString[] =
GUC_REPORT
},
&libpq_compress_algorithms,
- "off",
+ "on",
check_libpq_compression, assign_libpq_compression, NULL
},
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 0411571c02..24a4d6e05a 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -74,7 +74,7 @@
# (change requires restart)
#bonjour_name = '' # defaults to the computer name
# (change requires restart)
-#libpq_compression = off # on to allow all supported compression
+#libpq_compression = on # on to allow all supported compression
# methods; off to disable all compression;
# semicolon separated list of algorithms to allow some
diff --git a/src/common/zpq_stream.c b/src/common/zpq_stream.c
index 03ca8b7415..7c15ae89cf 100644
--- a/src/common/zpq_stream.c
+++ b/src/common/zpq_stream.c
@@ -221,7 +221,8 @@ zpq_choose_algorithm(ZpqStream * zpq, char msg_type, uint32 msg_len)
* has not yet been enabled) for message types that would most obviously
* benefit from compression
*/
- if (msg_len >= ZPQ_COMPRESS_THRESHOLD && (msg_type == PqMsg_CopyData || msg_type == PqMsg_DataRow || msg_type == PqMsg_Query))
+ /* force enable for testing */
+ if (true || (msg_len >= ZPQ_COMPRESS_THRESHOLD && (msg_type == PqMsg_CopyData || msg_type == PqMsg_DataRow || msg_type == PqMsg_Query)))
{
return zpq->compress_algs[0];
}
diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c
index 5e3e226457..5b2c29376f 100644
--- a/src/interfaces/libpq/fe-connect.c
+++ b/src/interfaces/libpq/fe-connect.c
@@ -399,7 +399,7 @@ static const internalPQconninfoOption PQconninfoOptions[] = {
"Replication", "D", 5,
offsetof(struct pg_conn, replication)},
- {"compression", "PGCOMPRESSION", "off", NULL,
+ {"compression", "PGCOMPRESSION", "on", NULL,
"Libpq-compression", "", 16,
offsetof(struct pg_conn, compression)},
@@ -1737,6 +1737,7 @@ connectOptions2(PGconn *conn)
return false;
}
+ /* Disable to allow SanityCheck to pass
if (rc == 1 && n_compressors == 0)
{
conn->status = CONNECTION_BAD;
@@ -1745,6 +1746,7 @@ connectOptions2(PGconn *conn)
"compression", conn->compression);
return false;
}
+ */
}
/*
--
2.42.0
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
@ 2024-01-12 20:45 ` Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
0 siblings, 1 reply; 46+ messages in thread
From: Robert Haas @ 2024-01-12 20:45 UTC (permalink / raw)
To: Jacob Burroughs <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; PostgreSQL Hackers <[email protected]>
On Sun, Dec 31, 2023 at 2:32 AM Jacob Burroughs
<[email protected]> wrote:
> I ended up reworking this to use a version of this option in place of
> the `none` hackery, but naming the parameters `compress` and
> `decompress, so to disable compression but allow decompression you
> would specify `libpq_compression=gzip:compress=off`.
I'm still a bit befuddled by this interface.
libpq_compression=gzip:compress=off looks a lot like it's saying "do
it, except don't". I guess that you're using "compress" and
"decompress" to distinguish the two directions - i.e. server to client
and client to server - but of course in both directions the sender
compresses and the receiver decompresses, so I don't find that very
clear.
I wonder if we could use "upstream" and "downstream" to be clearer? Or
some other terminology?
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
@ 2024-01-12 21:02 ` Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
0 siblings, 1 reply; 46+ messages in thread
From: Jacob Burroughs @ 2024-01-12 21:02 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; PostgreSQL Hackers <[email protected]>
> I wonder if we could use "upstream" and "downstream" to be clearer? Or
> some other terminology?
What about `send` and `receive`?
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
@ 2024-01-12 21:11 ` Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
0 siblings, 1 reply; 46+ messages in thread
From: Robert Haas @ 2024-01-12 21:11 UTC (permalink / raw)
To: Jacob Burroughs <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; PostgreSQL Hackers <[email protected]>
On Fri, Jan 12, 2024 at 4:02 PM Jacob Burroughs
<[email protected]> wrote:
> > I wonder if we could use "upstream" and "downstream" to be clearer? Or
> > some other terminology?
>
> What about `send` and `receive`?
I think that would definitely be better than "compress" and
"decompress," but I was worried that it might be unclear to the user
whether the parameter that they specified was from the point of view
of the client or the server. Perhaps that's a dumb thing to worry
about, though.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
@ 2024-05-14 16:08 ` Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
0 siblings, 1 reply; 46+ messages in thread
From: Robert Haas @ 2024-05-14 16:08 UTC (permalink / raw)
To: Jacob Burroughs <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; PostgreSQL Hackers <[email protected]>
On Fri, Jan 12, 2024 at 4:11 PM Robert Haas <[email protected]> wrote:
> I think that would definitely be better than "compress" and
> "decompress," but I was worried that it might be unclear to the user
> whether the parameter that they specified was from the point of view
> of the client or the server. Perhaps that's a dumb thing to worry
> about, though.
According to https://commitfest.postgresql.org/48/4746/ this patch set
needs review, but:
1. Considering that there have been no updates for 5 months, maybe
it's actually dead?
and
2. I still think it needs to be more clear how the interface is
supposed to work. I do not want to spend time reviewing a patch to see
whether it works without understanding how it is intended to work --
and I also think that reviewing the patch in detail before we've got
the user interface right makes a whole lot of sense.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
@ 2024-05-14 16:30 ` Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
0 siblings, 1 reply; 46+ messages in thread
From: Jacob Burroughs @ 2024-05-14 16:30 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; PostgreSQL Hackers <[email protected]>
On Tue, May 14, 2024 at 11:08 AM Robert Haas <[email protected]> wrote:
>
> According to https://commitfest.postgresql.org/48/4746/ this patch set
> needs review, but:
>
> 1. Considering that there have been no updates for 5 months, maybe
> it's actually dead?
I've withdrawn this patch from the commitfest. I had been waiting for
some resolution on "Add new protocol message to change GUCs for usage
with future protocol-only GUCs" before I rebased/refactored this one,
because this would be introducing the first protocol extension so far,
and that discussion appeared to be working out some meaningful issues
on how GUCs and protocol parameters should interact. If you think it
is worthwhile to proceed here though, I am very happy to do so. (I
would love to see this feature actually make it into postgres; it
would almost certainly be a big efficiency and cost savings win for
how my company deploys postgres internally :) )
> 2. I still think it needs to be more clear how the interface is
> supposed to work. I do not want to spend time reviewing a patch to see
> whether it works without understanding how it is intended to work --
> and I also think that reviewing the patch in detail before we've got
> the user interface right makes a whole lot of sense.
Regarding the interface, what I had originally gone for was the idea
that the naming of the options was from the perspective of the side
you were setting them on. Therefore, when setting `libpq_compression`
as a server-level GUC, `compress` would control if the server would
compress (send compressed data) with the given algorithm, and
`decompress` would control if the the server would decompress (receive
compressed data) with the given algorithm. And likewise on the client
side, when setting `compression` as a connection config option,
`compress` would control if the *client* would compress (send
compressed data) with the given algorithm, and `decompress` would
control if the the *client* would decompress (receive compressed data)
with the given algorithm. So for a client to pick what to send, it
would choose from the intersection of its own `compress=true` and the
server's `decompress=true` algorithms sent in the `ParameterStatus`
message with `libpq_compression`. And likewise on the server side, it
would choose from the intersection of the server's `compress=true`
algorithms and the client's `decompress=true` algorithms sent in the
`_pq_.libpq_compression` startup option. If you have a better
suggestion I am very open to it though.
--
Jacob Burroughs | Staff Software Engineer
E: [email protected]
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
@ 2024-05-14 18:35 ` Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
0 siblings, 1 reply; 46+ messages in thread
From: Robert Haas @ 2024-05-14 18:35 UTC (permalink / raw)
To: Jacob Burroughs <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; PostgreSQL Hackers <[email protected]>
On Tue, May 14, 2024 at 12:30 PM Jacob Burroughs
<[email protected]> wrote:
> I've withdrawn this patch from the commitfest. I had been waiting for
> some resolution on "Add new protocol message to change GUCs for usage
> with future protocol-only GUCs" before I rebased/refactored this one,
> because this would be introducing the first protocol extension so far,
> and that discussion appeared to be working out some meaningful issues
> on how GUCs and protocol parameters should interact. If you think it
> is worthwhile to proceed here though, I am very happy to do so. (I
> would love to see this feature actually make it into postgres; it
> would almost certainly be a big efficiency and cost savings win for
> how my company deploys postgres internally :) )
I don't think you should wait for that to be resolved; IMHO, this
patch needs to inform that discussion more than the other way around.
> > 2. I still think it needs to be more clear how the interface is
> > supposed to work. I do not want to spend time reviewing a patch to see
> > whether it works without understanding how it is intended to work --
> > and I also think that reviewing the patch in detail before we've got
> > the user interface right makes a whole lot of sense.
>
> Regarding the interface, what I had originally gone for was the idea
> that the naming of the options was from the perspective of the side
> you were setting them on. Therefore, when setting `libpq_compression`
> as a server-level GUC, `compress` would control if the server would
> compress (send compressed data) with the given algorithm, and
> `decompress` would control if the the server would decompress (receive
> compressed data) with the given algorithm. And likewise on the client
> side, when setting `compression` as a connection config option,
> `compress` would control if the *client* would compress (send
> compressed data) with the given algorithm, and `decompress` would
> control if the the *client* would decompress (receive compressed data)
> with the given algorithm. So for a client to pick what to send, it
> would choose from the intersection of its own `compress=true` and the
> server's `decompress=true` algorithms sent in the `ParameterStatus`
> message with `libpq_compression`. And likewise on the server side, it
> would choose from the intersection of the server's `compress=true`
> algorithms and the client's `decompress=true` algorithms sent in the
> `_pq_.libpq_compression` startup option. If you have a better
> suggestion I am very open to it though.
Well, in my last response before the thread died, I complained that
libpq_compression=gzip:compress=off was confusing, and I stand by
that, because "compress" is used both in the name of the parameter and
as an option within the value of that parameter. I think there's more
than one acceptable way to resolve that problem, but I think leaving
it like that is unacceptable.
Even more broadly, I think there have been a couple of versions of
this patch now where I read the documentation and couldn't understand
how the feature was supposed to work, and I'm not really willing to
spend time trying to review a complex patch for conformity with a
design that I can't understand in the first place. I don't want to
pretend like I'm the smartest person on this mailing list, and in fact
I know that I'm definitely not, but I think I'm smart enough and
experienced enough with PostgreSQL that if I look at the description
of a parameter and say "I don't understand how the heck this is
supposed to work", probably a lot of users are going to have the same
reaction. That lack of understanding on my part my come either from
the explanation of the parameter not being as good as it needs to be,
or from the design itself not being as good as it needs to be, or from
some combination of the two, but whichever is the case, IMHO you or
somebody else has got to figure out how to fix it.
I do also admit that there is a possibility that everything is totally
fine and I've just been kinda dumb on the days when I've looked at the
patch. If a chorus of other hackers shows up and gives me a few whacks
with the cluestick and after that I look at the proposed options and
go "oh, yeah, these totally make sense, I was just being stupid," fair
enough! But right now that's not where I'm at. I don't want you to
explain to me how it works; I want you to change it in some way so
that when I or some end user looks at it, they go "I don't need an
explanation of how that works because it's extremely clear to me
already," or at least "hmm, this is a bit complicated but after a
quick glance at the documentation it makes sense".
I would really like to see this patch go forward, but IMHO these UI
questions are blockers.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
@ 2024-05-14 19:22 ` Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
0 siblings, 1 reply; 46+ messages in thread
From: Jacob Burroughs @ 2024-05-14 19:22 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; PostgreSQL Hackers <[email protected]>
On Tue, May 14, 2024 at 1:35 PM Robert Haas <[email protected]> wrote:
>
> Well, in my last response before the thread died, I complained that
> libpq_compression=gzip:compress=off was confusing, and I stand by
> that, because "compress" is used both in the name of the parameter and
> as an option within the value of that parameter. I think there's more
> than one acceptable way to resolve that problem, but I think leaving
> it like that is unacceptable.
What if we went with:
Server side:
* `libpq_compression=on` (I just want everything the server supports
available; probably the most common case)
* `libpq_compression=off` (I don't want any compression ever with this server)
* `libpq_compression=lzma;gzip` (I only want these algorithms for
whatever reason)
* `libpq_compression=lzma:client_to_server=off;gzip:server_to_client=off`
(I only want to send data with lzma and receive data with gzip)
Client side:
*`compression=on` (I just want compression; pick sane defaults
automatically for me; probably the most common case)
* `compression=off` (I don't want any compression)
* `compression=lzma;gzip` (I only want these algorithms for whatever reason)
* `compression=lzma:client_to_server=off;gzip:server_to_client=off` (I
only want to receive data with lzma and send data with gzip)
`client_to_server`/`server_to_client` is a bit verbose, but it's very
explicit in what it means, so you don't need to reason about who is
sending/receiving/etc in a given context, and a given config string
applied to the server or the client side has the same effect on the
connection.
> Even more broadly, I think there have been a couple of versions of
> this patch now where I read the documentation and couldn't understand
> how the feature was supposed to work, and I'm not really willing to
> spend time trying to review a complex patch for conformity with a
> design that I can't understand in the first place. I don't want to
> pretend like I'm the smartest person on this mailing list, and in fact
> I know that I'm definitely not, but I think I'm smart enough and
> experienced enough with PostgreSQL that if I look at the description
> of a parameter and say "I don't understand how the heck this is
> supposed to work", probably a lot of users are going to have the same
> reaction. That lack of understanding on my part my come either from
> the explanation of the parameter not being as good as it needs to be,
> or from the design itself not being as good as it needs to be, or from
> some combination of the two, but whichever is the case, IMHO you or
> somebody else has got to figure out how to fix it.
If the above proposal seems better to you I'll both rework the patch
and then also try to rewrite the relevant bits of documentation to
separate out "what knobs are there" and "how do I specify the flags to
turn the knobs", because I think those two being integrated is making
the parameter documentation less readable/followable.
--
Jacob Burroughs | Staff Software Engineer
E: [email protected]
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
@ 2024-05-14 20:23 ` Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
0 siblings, 1 reply; 46+ messages in thread
From: Robert Haas @ 2024-05-14 20:23 UTC (permalink / raw)
To: Jacob Burroughs <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; PostgreSQL Hackers <[email protected]>
On Tue, May 14, 2024 at 3:22 PM Jacob Burroughs
<[email protected]> wrote:
> What if we went with:
> Server side:
> * `libpq_compression=on` (I just want everything the server supports
> available; probably the most common case)
> * `libpq_compression=off` (I don't want any compression ever with this server)
> * `libpq_compression=lzma;gzip` (I only want these algorithms for
> whatever reason)
> * `libpq_compression=lzma:client_to_server=off;gzip:server_to_client=off`
> (I only want to send data with lzma and receive data with gzip)
> Client side:
> *`compression=on` (I just want compression; pick sane defaults
> automatically for me; probably the most common case)
> * `compression=off` (I don't want any compression)
> * `compression=lzma;gzip` (I only want these algorithms for whatever reason)
> * `compression=lzma:client_to_server=off;gzip:server_to_client=off` (I
> only want to receive data with lzma and send data with gzip)
>
> `client_to_server`/`server_to_client` is a bit verbose, but it's very
> explicit in what it means, so you don't need to reason about who is
> sending/receiving/etc in a given context, and a given config string
> applied to the server or the client side has the same effect on the
> connection.
IMHO, that's a HUGE improvement. But:
* I would probably change is the name "libpq_compression", because
even though we have src/backend/libpq, we typically use libpq to refer
to the client library, not the server's implementation of the wire
protocol. I think we could call it connection_encryption or
wire_protocol_encryption or something like that, but I'm not a huge
fan of libpq_compression.
* I would use commas, not semicolons, to separate items in a list,
i.e. lzma,gzip not lzma;gzip. I think that convention is nearly
universal in PostgreSQL, but feel free to point out counterexamples if
you were modelling this on something.
* libpq_compression=lzma:client_to_server=off;gzip:server_to_client=off
reads strangely to me. How about making it so that the syntax is like
this:
libpq_compression=DEFAULT_VALUE_FOR_BOTH_DIRECTIONS:client_to_server=OVERRIDE_FOR_THIS_DIRECTION:servert_to_client=OVERRIDE_FOR_THIS_DIRECTION
With all components being optional. So this example could be written
in any of these ways:
libpq_compression=lzma;server_to_client=gzip
libpq_compression=gzip;client_to_server=lzma
libpq_compression=server_to_client=gzip;client_to_server=lzma
libpq_compression=client_to_server=lzma;client_to_server=gzip
And if I wrote libpq_compression=server_to_client=gzip that would mean
send data to the client using gzip and in the other direction use
whatever the default is.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 46+ messages in thread
* Fwd: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
@ 2024-05-14 21:21 ` Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
0 siblings, 1 reply; 46+ messages in thread
From: Jacob Burroughs @ 2024-05-14 21:21 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; PostgreSQL Hackers <[email protected]>
On Tue, May 14, 2024 at 3:24 PM Robert Haas <[email protected]> wrote:
>
> IMHO, that's a HUGE improvement. But:
>
> * I would probably change is the name "libpq_compression", because
> even though we have src/backend/libpq, we typically use libpq to refer
> to the client library, not the server's implementation of the wire
> protocol. I think we could call it connection_encryption or
> wire_protocol_encryption or something like that, but I'm not a huge
> fan of libpq_compression.
>
I think connection_compression would seem like a good name to me.
> * I would use commas, not semicolons, to separate items in a list,
> i.e. lzma,gzip not lzma;gzip. I think that convention is nearly
> universal in PostgreSQL, but feel free to point out counterexamples if
> you were modelling this on something.
>
> * libpq_compression=lzma:client_to_server=off;gzip:server_to_client=off
> reads strangely to me. How about making it so that the syntax is like
> this:
>
> libpq_compression=DEFAULT_VALUE_FOR_BOTH_DIRECTIONS:client_to_server=OVERRIDE_FOR_THIS_DIRECTION:servert_to_client=OVERRIDE_FOR_THIS_DIRECTION
>
> With all components being optional. So this example could be written
> in any of these ways:
>
> libpq_compression=lzma;server_to_client=gzip
> libpq_compression=gzip;client_to_server=lzma
> libpq_compression=server_to_client=gzip;client_to_server=lzma
> libpq_compression=client_to_server=lzma;client_to_server=gzip
>
> And if I wrote libpq_compression=server_to_client=gzip that would mean
> send data to the client using gzip and in the other direction use
> whatever the default is.
The reason for both the semicolons and for not doing this is related
to using the same specification structure as here:
https://www.postgresql.org/docs/current/app-pgbasebackup.html
(specifically the --compress argument). Reusing that specification
requires that we use commas to separate the flags for a compression
method, and therefore left me with semicolons as the leftover
separator character. I think we could go with something like your
proposal, and in a lot of ways I like it, but there's still the
possibility of e.g.
`libpq_compression=client_to_server=zstd:level=10,long=true,gzip;client_to_server=gzip`
and I'm not quite sure how to make the separator characters work
coherently if we need to treat `zstd:level=10,long=true` as a unit.
Alternatively, we could have `connection_compression`,
`connection_compression_server_to_client`, and
`connection_compression_client_to_server` as three separate GUCs (and
on the client side `compression`, `compression_server_to_client`, and
`compression_client_to_server` as three separate connection
parameters), where we would treat `connection_compression` as a
default that could be overridden by an explicit
client_to_server/server_to_client. That creates the slightly funky
case where if you specify all three then the base one ends up unused
because the two more specific ones are being used instead, but that
isn't necessarily terrible. On the server side we *could* go with
just the server_to_client and client_to_server ones, but I think we
want it to be easy to use this feature in the simple case with a
single libpq parameter.
--
Jacob Burroughs | Staff Software Engineer
E: [email protected]
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
@ 2024-05-15 13:38 ` Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
0 siblings, 1 reply; 46+ messages in thread
From: Robert Haas @ 2024-05-15 13:38 UTC (permalink / raw)
To: Jacob Burroughs <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; PostgreSQL Hackers <[email protected]>
On Tue, May 14, 2024 at 5:21 PM Jacob Burroughs
<[email protected]> wrote:
> The reason for both the semicolons and for not doing this is related
> to using the same specification structure as here:
> https://www.postgresql.org/docs/current/app-pgbasebackup.html
> (specifically the --compress argument).
I agree with that goal, but I'm somewhat confused by how your proposal
achieves it. You had
libpq_compression=lzma:client_to_server=off;gzip:server_to_client=off,
so how do we parse that? Is that two completely separate
specifications, one for lzma and one for gzip, and each of those has
one option which is set to off? And then they are separated from each
other by a semicolon? That actually does make sense, and I think it
may do a better job allowing for compression options than my proposal,
but it also seems a bit weird, because client_to_server and
server_to_client are not really compression options at all. They're
framing when this compression specification applies, rather than what
it does when it applies. In a way it's a bit like the fact that you
can prefix a pg_basebackup's --compress option with client- or server-
to specify where the compression should happen. But we can't quite
reuse that idea here, because in that case there's no question of
doing it in both places, whereas here, you might want one thing for
upstream and another thing for downstream.
> Alternatively, we could have `connection_compression`,
> `connection_compression_server_to_client`, and
> `connection_compression_client_to_server` as three separate GUCs (and
> on the client side `compression`, `compression_server_to_client`, and
> `compression_client_to_server` as three separate connection
> parameters), where we would treat `connection_compression` as a
> default that could be overridden by an explicit
> client_to_server/server_to_client. That creates the slightly funky
> case where if you specify all three then the base one ends up unused
> because the two more specific ones are being used instead, but that
> isn't necessarily terrible. On the server side we *could* go with
> just the server_to_client and client_to_server ones, but I think we
> want it to be easy to use this feature in the simple case with a
> single libpq parameter.
I'm not a fan of three settings; I could go with two settings, one for
each direction, and if you want both you have to set both. Or, another
idea, what if we just separated the two directions with a slash,
SEND/RECEIVE, and if there's no slash, then it applies to both
directions. So you could say
connection_compression='gzip:level=9/lzma' or whatever.
But now I'm wondering whether these options should really be symmetric
on the client and server sides? Isn't it for the server just to
specify a list of acceptable algorithms, and the client to set the
compression options? If both sides are trying to set the compression
level, for example, who wins?
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
@ 2024-05-15 16:24 ` Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
0 siblings, 1 reply; 46+ messages in thread
From: Jacob Burroughs @ 2024-05-15 16:24 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; PostgreSQL Hackers <[email protected]>
On Wed, May 15, 2024 at 8:38 AM Robert Haas <[email protected]> wrote:
>
> I agree with that goal, but I'm somewhat confused by how your proposal
> achieves it. You had
> libpq_compression=lzma:client_to_server=off;gzip:server_to_client=off,
> so how do we parse that? Is that two completely separate
> specifications, one for lzma and one for gzip, and each of those has
> one option which is set to off? And then they are separated from each
> other by a semicolon? That actually does make sense, and I think it
> may do a better job allowing for compression options than my proposal,
> but it also seems a bit weird, because client_to_server and
> server_to_client are not really compression options at all. They're
> framing when this compression specification applies, rather than what
> it does when it applies. In a way it's a bit like the fact that you
> can prefix a pg_basebackup's --compress option with client- or server-
> to specify where the compression should happen. But we can't quite
> reuse that idea here, because in that case there's no question of
> doing it in both places, whereas here, you might want one thing for
> upstream and another thing for downstream.
Your interpretation is correct, but I don't disagree that it ends up
feeling confusing.
> I'm not a fan of three settings; I could go with two settings, one for
> each direction, and if you want both you have to set both. Or, another
> idea, what if we just separated the two directions with a slash,
> SEND/RECEIVE, and if there's no slash, then it applies to both
> directions. So you could say
> connection_compression='gzip:level=9/lzma' or whatever.
>
> But now I'm wondering whether these options should really be symmetric
> on the client and server sides? Isn't it for the server just to
> specify a list of acceptable algorithms, and the client to set the
> compression options? If both sides are trying to set the compression
> level, for example, who wins?
Compression options really only ever apply to the side doing the
compressing, and at least as I had imagined things each party
(client/server) only used its own level/other compression params.
That leaves me thinking, maybe we really want two independent GUCs,
one for "what algorithms are enabled/negotiable" and one for "how
should I configure my compressors" and then we reduce the dimensions
we are trying to shove into one GUC and each one ends up with a very
clear purpose:
connection_compression=(yes|no|alg1,alg2:server_to_client=alg1,alg2:client_to_server=alg3)
connection_compression_opts=gzip:level=2
--
Jacob Burroughs | Staff Software Engineer
E: [email protected]
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
@ 2024-05-15 16:31 ` Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
0 siblings, 1 reply; 46+ messages in thread
From: Robert Haas @ 2024-05-15 16:31 UTC (permalink / raw)
To: Jacob Burroughs <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; PostgreSQL Hackers <[email protected]>
On Wed, May 15, 2024 at 12:24 PM Jacob Burroughs
<[email protected]> wrote:
> > But now I'm wondering whether these options should really be symmetric
> > on the client and server sides? Isn't it for the server just to
> > specify a list of acceptable algorithms, and the client to set the
> > compression options? If both sides are trying to set the compression
> > level, for example, who wins?
>
> Compression options really only ever apply to the side doing the
> compressing, and at least as I had imagined things each party
> (client/server) only used its own level/other compression params.
> That leaves me thinking, maybe we really want two independent GUCs,
> one for "what algorithms are enabled/negotiable" and one for "how
> should I configure my compressors" and then we reduce the dimensions
> we are trying to shove into one GUC and each one ends up with a very
> clear purpose:
> connection_compression=(yes|no|alg1,alg2:server_to_client=alg1,alg2:client_to_server=alg3)
> connection_compression_opts=gzip:level=2
From my point of view, it's the client who knows what it wants to do
with the connection. If the client plans to read a lot of data, it
might want the server to compress that data, especially if it knows
that it's on a slow link. If the client plans to send a lot of data --
basically COPY, I'm not thinking this is going to matter much
otherwise -- then it might want to compress that data before sending
it, again, especially if it knows that it's on a slow link.
But what does the server know, really? If some client connects and
sends a SELECT query, the server can't guess whether that query is
going to return 1 row or 100 million rows, so it has no idea of
whether compression is likely to make sense or not. It is entitled to
decide, as a matter of policy, that it's not willing to perform
compression, either because of CPU consumption or security concerns or
whatever, but it has no knowledge of what the purpose of this
particular connection is.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
@ 2024-05-15 16:50 ` Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
0 siblings, 1 reply; 46+ messages in thread
From: Jacob Burroughs @ 2024-05-15 16:50 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; PostgreSQL Hackers <[email protected]>
On Wed, May 15, 2024 at 11:31 AM Robert Haas <[email protected]> wrote:
> From my point of view, it's the client who knows what it wants to do
> with the connection. If the client plans to read a lot of data, it
> might want the server to compress that data, especially if it knows
> that it's on a slow link. If the client plans to send a lot of data --
> basically COPY, I'm not thinking this is going to matter much
> otherwise -- then it might want to compress that data before sending
> it, again, especially if it knows that it's on a slow link.
>
> But what does the server know, really? If some client connects and
> sends a SELECT query, the server can't guess whether that query is
> going to return 1 row or 100 million rows, so it has no idea of
> whether compression is likely to make sense or not. It is entitled to
> decide, as a matter of policy, that it's not willing to perform
> compression, either because of CPU consumption or security concerns or
> whatever, but it has no knowledge of what the purpose of this
> particular connection is.
I think I would agree with that. That said, I don't think the client
should be in the business of specifying what configuration of the
compression algorithm the server should use. The server administrator
(or really most of the time, the compression library developer's
defaults) gets to pick the compression/compute tradeoff for
compression that runs on the server (which I would imagine would be
the vast majority of it), and the client gets to pick those same
parameters for any compression that runs on the client machine
(probably indeed in practice only for large COPYs). The *algorithm*
needs to actually be communicated/negotiated since different
client/server pairs may be built with support for different
compression libraries, but I think it is reasonable to say that the
side that actually has to do the computationally expensive part owns
the configuration of that part too. Maybe I'm missing a good reason
that we want to allow clients to choose compression levels for the
server though?
--
Jacob Burroughs | Staff Software Engineer
E: [email protected]
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
@ 2024-05-16 13:28 ` Robert Haas <[email protected]>
2024-05-17 20:53 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
0 siblings, 1 reply; 46+ messages in thread
From: Robert Haas @ 2024-05-16 13:28 UTC (permalink / raw)
To: Jacob Burroughs <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; PostgreSQL Hackers <[email protected]>
On Wed, May 15, 2024 at 12:50 PM Jacob Burroughs
<[email protected]> wrote:
> I think I would agree with that. That said, I don't think the client
> should be in the business of specifying what configuration of the
> compression algorithm the server should use. The server administrator
> (or really most of the time, the compression library developer's
> defaults) gets to pick the compression/compute tradeoff for
> compression that runs on the server (which I would imagine would be
> the vast majority of it), and the client gets to pick those same
> parameters for any compression that runs on the client machine
> (probably indeed in practice only for large COPYs). The *algorithm*
> needs to actually be communicated/negotiated since different
> client/server pairs may be built with support for different
> compression libraries, but I think it is reasonable to say that the
> side that actually has to do the computationally expensive part owns
> the configuration of that part too. Maybe I'm missing a good reason
> that we want to allow clients to choose compression levels for the
> server though?
Well, I mean, I don't really know what the right answer is here, but
right now I can say pg_dump --compress=gzip to compress the dump with
gzip, or pg_dump --compress=gzip:9 to compress with gzip level 9. Now,
say that instead of compressing the output, I want to compress the
data sent to me over the connection. So I figure I should be able to
say pg_dump 'compress=gzip' or pg_dump 'compress=gzip:9'. I think you
want to let me do the first of those but not the second. But, to turn
your question on its head, what would be the reasoning behind such a
restriction?
Note also the precedent of pg_basebackup. I can say pg_basebackup
--compress=server-gzip:9 to ask the server to compress the backup with
gzip at level 9. In that case, what I request from the server changes
the actual output that I get, which is not the case here. Even so, I
don't really understand what the justification would be for refusing
to let the client ask for a specific compression level.
And on the flip side, I also don't understand why the server would
want to mandate a certain compression level. If compression is very
expensive for a certain algorithm when the level is above some
threshold X, we could have a GUC to limit the maximum level that the
client can request. But, given that the gzip compression level
defaults to 6 in every other context, why would the administrator of a
particular server want to say, well, the default for my server is 3 or
9 or whatever?
(This is of course all presuming you want to use gzip at all, which
you probably don't, because gzip is crazy slow. Use lz4 or zstd! But
it makes the point.)
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
@ 2024-05-17 20:53 ` Jacob Burroughs <[email protected]>
2024-05-17 21:10 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-17 21:40 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
0 siblings, 2 replies; 46+ messages in thread
From: Jacob Burroughs @ 2024-05-17 20:53 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; PostgreSQL Hackers <[email protected]>
On Thu, May 16, 2024 at 3:28 AM Robert Haas <[email protected]> wrote:
>
> Well, I mean, I don't really know what the right answer is here, but
> right now I can say pg_dump --compress=gzip to compress the dump with
> gzip, or pg_dump --compress=gzip:9 to compress with gzip level 9. Now,
> say that instead of compressing the output, I want to compress the
> data sent to me over the connection. So I figure I should be able to
> say pg_dump 'compress=gzip' or pg_dump 'compress=gzip:9'. I think you
> want to let me do the first of those but not the second. But, to turn
> your question on its head, what would be the reasoning behind such a
> restriction?
I think I was more thinking that trying to let both parties control
the parameter seemed like a recipe for confusion and sadness, and so
the choice that felt most natural to me was to let the sender control
it, but I'm definitely open to changing that the other way around.
> Note also the precedent of pg_basebackup. I can say pg_basebackup
> --compress=server-gzip:9 to ask the server to compress the backup with
> gzip at level 9. In that case, what I request from the server changes
> the actual output that I get, which is not the case here. Even so, I
> don't really understand what the justification would be for refusing
> to let the client ask for a specific compression level.
>
> And on the flip side, I also don't understand why the server would
> want to mandate a certain compression level. If compression is very
> expensive for a certain algorithm when the level is above some
> threshold X, we could have a GUC to limit the maximum level that the
> client can request. But, given that the gzip compression level
> defaults to 6 in every other context, why would the administrator of a
> particular server want to say, well, the default for my server is 3 or
> 9 or whatever?
>
> (This is of course all presuming you want to use gzip at all, which
> you probably don't, because gzip is crazy slow. Use lz4 or zstd! But
> it makes the point.)
New proposal, predicated on the assumption that if you enable
compression you are ok with the client picking whatever level they
want. At least with the currently enabled algorithms I don't think
any of them are so insane that they would knock over a server or
anything, and in general postgres servers are usually connected to by
clients that the server admin has some channel to talk to (after all
they somehow had to get access to log in to the server in the first
place) if they are doing something wasteful, given that a client can
do a lot worse things than enable aggressive compression by writing
bad queries.
On the server side, we use slash separated sets of options
connection_compression=DEFAULT_VALUE_FOR_BOTH_DIRECTIONS/client_to_server=OVERRIDE_FOR_THIS_DIRECTION/server_to_client=OVERRIDE_FOR_THIS_DIRECTION
with the values being semicolon separated compression algorithms.
On the client side, you can specify
compression=<same_specification_as_above>,
but on the client side you can actually specify compression options,
which the server will use if provided, and otherwise it will fall back
to defaults.
If we think we need to, we could let the server specify defaults for
server-side compression. My overall thought though is that having an
excessive number of knobs increases the surface area for testing and
bugs while also increasing potential user confusion and that allowing
configuration on *both* sides doesn't seem sufficiently useful to be
worth adding that complexity.
--
Jacob Burroughs | Staff Software Engineer
E: [email protected]
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-17 20:53 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
@ 2024-05-17 21:10 ` Jacob Champion <[email protected]>
2024-05-17 23:02 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
1 sibling, 1 reply; 46+ messages in thread
From: Jacob Champion @ 2024-05-17 21:10 UTC (permalink / raw)
To: Jacob Burroughs <[email protected]>; +Cc: Robert Haas <[email protected]>; Jelte Fennema-Nio <[email protected]>; PostgreSQL Hackers <[email protected]>
On Fri, May 17, 2024 at 1:53 PM Jacob Burroughs
<[email protected]> wrote:
> New proposal, predicated on the assumption that if you enable
> compression you are ok with the client picking whatever level they
> want. At least with the currently enabled algorithms I don't think
> any of them are so insane that they would knock over a server or
> anything, and in general postgres servers are usually connected to by
> clients that the server admin has some channel to talk to (after all
> they somehow had to get access to log in to the server in the first
> place) if they are doing something wasteful, given that a client can
> do a lot worse things than enable aggressive compression by writing
> bad queries.
We're talking about a transport-level option, though -- I thought the
proposal enabled compression before authentication completed? Or has
that changed?
(I'm suspicious of arguments that begin "well you can already do bad
things", anyway... It seems like there's a meaningful difference
between consuming resources running a parsed query and consuming
resources trying to figure out what the parsed query is. I don't know
if the solution is locking in a compression level, or something else;
maybe they're both reasonably mitigated in the same way. I haven't
really looked into zip bombs much.)
--Jacob
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-17 20:53 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-17 21:10 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
@ 2024-05-17 23:02 ` Jelte Fennema-Nio <[email protected]>
2024-05-18 05:18 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-20 14:14 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
0 siblings, 2 replies; 46+ messages in thread
From: Jelte Fennema-Nio @ 2024-05-17 23:02 UTC (permalink / raw)
To: Jacob Champion <[email protected]>; +Cc: Jacob Burroughs <[email protected]>; Robert Haas <[email protected]>; PostgreSQL Hackers <[email protected]>
On Fri, 17 May 2024 at 23:10, Jacob Champion
<[email protected]> wrote:
> We're talking about a transport-level option, though -- I thought the
> proposal enabled compression before authentication completed? Or has
> that changed?
I think it would make sense to only compress messages after
authentication has completed. The gain of compressing authentication
related packets seems pretty limited.
> (I'm suspicious of arguments that begin "well you can already do bad
> things"
Once logged in it's really easy to max out a core of the backend
you're connected as. There's many trivial queries you can use to do
that. An example would be:
SELECT sum(i) from generate_series(1, 1000000000) i;
So I don't think it makes sense to worry about an attacker using a
high compression level as a means to DoS the server. Sending a few of
the above queries seems much easier.
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-17 20:53 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-17 21:10 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-17 23:02 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
@ 2024-05-18 05:18 ` Jacob Burroughs <[email protected]>
2024-05-20 15:17 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
1 sibling, 1 reply; 46+ messages in thread
From: Jacob Burroughs @ 2024-05-18 05:18 UTC (permalink / raw)
To: Jelte Fennema-Nio <[email protected]>; +Cc: Jacob Champion <[email protected]>; Robert Haas <[email protected]>; PostgreSQL Hackers <[email protected]>
On Fri, May 17, 2024 at 11:40 AM Robert Haas <[email protected]> wrote:
>
> To be clear, I am not arguing that it should be the receiver's choice.
> I'm arguing it should be the client's choice, which means the client
> decides what it sends and also tells the server what to send to it.
> I'm open to counter-arguments, but as I've thought about this more,
> I've come to the conclusion that letting the client control the
> behavior is the most likely to be useful and the most consistent with
> existing facilities. I think we're on the same page based on the rest
> of your email: I'm just clarifying.
This is what I am imagining too
> I have some quibbles with the syntax but I agree with the concept.
> What I'd probably do is separate the server side thing into two GUCs,
> each with a list of algorithms, comma-separated, like we do for other
> lists in postgresql.conf. Maybe make the default 'all' meaning
> "everything this build of the server supports". On the client side,
> I'd allow both things to be specified using a single option, because
> wanting to do the same thing in both directions will be common, and
> you actually have to type in connection strings sometimes, so
> verbosity matters more.
>
> As far as the format of the value for that keyword, what do you think
> about either compression=DO_THIS_BOTH_WAYS or
> compression=DO_THIS_WHEN_SENDING/DO_THIS_WHEN_RECEIVING, with each "do
> this" being a specification of the same form already accepted for
> server-side compression e.g. gzip or gzip:level=9? If you don't like
> that, why do you think the proposal you made above is better, and why
> is that one now punctuated with slashes instead of semicolons?
I like this more than what I proposed, and will update the patches to
reflect this proposal. (I've gotten them locally back into a state of
applying cleanly and dealing with the changes needed to support direct
SSL connections, so refactoring the protocol layer shouldn't be too
hard now.)
On Fri, May 17, 2024 at 11:10 AM Jacob Champion
<[email protected]> wrote:
> We're talking about a transport-level option, though -- I thought the
> proposal enabled compression before authentication completed? Or has
> that changed?
On Fri, May 17, 2024 at 1:03 PM Jelte Fennema-Nio <[email protected]> wrote:
> I think it would make sense to only compress messages after
> authentication has completed. The gain of compressing authentication
> related packets seems pretty limited.
At the protocol level, compressed data is a message type that can be
used to wrap arbitrary data as soon as the startup packet is
processed. However, as an implementation detail that clients should
not rely on but that we can rely on in thinking about the
implications, the only message types that are compressed (except in
the 0005 CI patch for test running only) are PqMsg_CopyData,
PqMsg_DataRow, and PqMsg_Query, all of which aren't sent before
authentication.
--
Jacob Burroughs | Staff Software Engineer
E: [email protected]
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-17 20:53 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-17 21:10 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-17 23:02 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2024-05-18 05:18 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
@ 2024-05-20 15:17 ` Robert Haas <[email protected]>
0 siblings, 0 replies; 46+ messages in thread
From: Robert Haas @ 2024-05-20 15:17 UTC (permalink / raw)
To: Jacob Burroughs <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; Jacob Champion <[email protected]>; PostgreSQL Hackers <[email protected]>
On Sat, May 18, 2024 at 1:18 AM Jacob Burroughs
<[email protected]> wrote:
> I like this more than what I proposed, and will update the patches to
> reflect this proposal. (I've gotten them locally back into a state of
> applying cleanly and dealing with the changes needed to support direct
> SSL connections, so refactoring the protocol layer shouldn't be too
> hard now.)
Sounds good!
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-17 20:53 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-17 21:10 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-17 23:02 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
@ 2024-05-20 14:14 ` Jacob Champion <[email protected]>
2024-05-20 15:29 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
1 sibling, 1 reply; 46+ messages in thread
From: Jacob Champion @ 2024-05-20 14:14 UTC (permalink / raw)
To: Jelte Fennema-Nio <[email protected]>; +Cc: Jacob Burroughs <[email protected]>; Robert Haas <[email protected]>; PostgreSQL Hackers <[email protected]>
On Fri, May 17, 2024 at 4:03 PM Jelte Fennema-Nio <[email protected]> wrote:
>
> On Fri, 17 May 2024 at 23:10, Jacob Champion
> <[email protected]> wrote:
> > We're talking about a transport-level option, though -- I thought the
> > proposal enabled compression before authentication completed? Or has
> > that changed?
>
> I think it would make sense to only compress messages after
> authentication has completed. The gain of compressing authentication
> related packets seems pretty limited.
Okay. But if we're relying on that for its security properties, it
needs to be enforced by the server.
> > (I'm suspicious of arguments that begin "well you can already do bad
> > things"
>
> Once logged in it's really easy to max out a core of the backend
> you're connected as. There's many trivial queries you can use to do
> that. An example would be:
> SELECT sum(i) from generate_series(1, 1000000000) i;
This is just restating the "you can already do bad things" argument. I
understand that if your query gets executed, it's going to consume
resources on the thing that's executing it (for the record, though,
there are people working on constraining that). But introducing
disproportionate resource consumption into all traffic-inspecting
software, like pools and bouncers, seems like a different thing to me.
Many use cases are going to be fine with it, of course, but I don't
think it should be hand-waved.
--Jacob
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-17 20:53 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-17 21:10 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-17 23:02 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2024-05-20 14:14 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
@ 2024-05-20 15:29 ` Robert Haas <[email protected]>
2024-05-20 16:49 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
0 siblings, 1 reply; 46+ messages in thread
From: Robert Haas @ 2024-05-20 15:29 UTC (permalink / raw)
To: Jacob Champion <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; Jacob Burroughs <[email protected]>; PostgreSQL Hackers <[email protected]>
On Mon, May 20, 2024 at 10:15 AM Jacob Champion
<[email protected]> wrote:
> This is just restating the "you can already do bad things" argument. I
> understand that if your query gets executed, it's going to consume
> resources on the thing that's executing it (for the record, though,
> there are people working on constraining that). But introducing
> disproportionate resource consumption into all traffic-inspecting
> software, like pools and bouncers, seems like a different thing to me.
> Many use cases are going to be fine with it, of course, but I don't
> think it should be hand-waved.
I can't follow this argument.
I think it's important that the startup message is always sent
uncompressed, because it's a strange exception to our usual
message-formatting rules, and because it's so security-critical. I
don't think we should do anything to allow more variation there,
because any benefit will be small and the chances of introducing
security vulnerabilities seems non-trivial.
But if the client says in the startup message that it would like to
send and receive compressed data and the server is happy with that
request, I don't see why we need to postpone implementing that request
until after the authentication exchange is completed. I think that
will make the code more complicated and I don't see a security
benefit. If the use of a particular compression algorithm is going to
impose too much load, the server, or the pooler, is free to refuse it,
and should. Deferring the use of the compression method until after
authentication doesn't really solve any problem here, at least not
that I can see.
It does occur to me that if some compression algorithm has a buffer
overrun bug, restricting its use until after authentication might
reduce the score of the resulting CVE, because now you have to be able
to authenticate to make an exploit work. Perhaps that's an argument
for imposing a restriction here, but it doesn't seem to be the
argument that you're making.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-17 20:53 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-17 21:10 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-17 23:02 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2024-05-20 14:14 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 15:29 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
@ 2024-05-20 16:49 ` Jacob Champion <[email protected]>
2024-05-20 17:01 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
0 siblings, 1 reply; 46+ messages in thread
From: Jacob Champion @ 2024-05-20 16:49 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; Jacob Burroughs <[email protected]>; PostgreSQL Hackers <[email protected]>
On Mon, May 20, 2024 at 8:29 AM Robert Haas <[email protected]> wrote:
> It does occur to me that if some compression algorithm has a buffer
> overrun bug, restricting its use until after authentication might
> reduce the score of the resulting CVE, because now you have to be able
> to authenticate to make an exploit work. Perhaps that's an argument
> for imposing a restriction here, but it doesn't seem to be the
> argument that you're making.
It wasn't my argument; Jacob B said above:
> in general postgres servers are usually connected to by
> clients that the server admin has some channel to talk to (after all
> they somehow had to get access to log in to the server in the first
> place) if they are doing something wasteful, given that a client can
> do a lot worse things than enable aggressive compression by writing
> bad queries.
...and my response was that, no, the proposal doesn't seem to be
requiring that authentication take place before compression is done.
(As evidenced by your email. :D) If the claim is that there are no
security problems with letting unauthenticated clients force
decompression, then I can try to poke holes in that; or if the claim
is that we don't need to worry about that at all because we'll wait
until after authentication, then I can poke holes in that too. My
request is just that we choose one.
> But if the client says in the startup message that it would like to
> send and receive compressed data and the server is happy with that
> request, I don't see why we need to postpone implementing that request
> until after the authentication exchange is completed. I think that
> will make the code more complicated and I don't see a security
> benefit.
I haven't implemented compression bombs before to know lots of
details, but I think the general idea is to take up resources that are
vastly disproportionate to the effort expended by the client. The
systemic risk is then more or less multiplied by the number of
intermediaries that need to do the decompression. Maybe all three of
our algorithms are hardened against malicious compression techniques;
that'd be great. But if we've never had a situation where a completely
untrusted peer can hand a blob to the server and say "here, decompress
this for me", maybe we'd better check?
--Jacob
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-17 20:53 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-17 21:10 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-17 23:02 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2024-05-20 14:14 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 15:29 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 16:49 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
@ 2024-05-20 17:01 ` Robert Haas <[email protected]>
2024-05-20 17:22 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
0 siblings, 1 reply; 46+ messages in thread
From: Robert Haas @ 2024-05-20 17:01 UTC (permalink / raw)
To: Jacob Champion <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; Jacob Burroughs <[email protected]>; PostgreSQL Hackers <[email protected]>
On Mon, May 20, 2024 at 12:49 PM Jacob Champion
<[email protected]> wrote:
> ...and my response was that, no, the proposal doesn't seem to be
> requiring that authentication take place before compression is done.
> (As evidenced by your email. :D) If the claim is that there are no
> security problems with letting unauthenticated clients force
> decompression, then I can try to poke holes in that;
I would prefer this approach, so I suggest trying to poke holes here
first. If you find big enough holes then...
> or if the claim
> is that we don't need to worry about that at all because we'll wait
> until after authentication, then I can poke holes in that too. My
> request is just that we choose one.
...we can fall back to this and you can try to poke holes here.
I really hope that you can't poke big enough holes to kill the feature
entirely, though. Because that sounds sad.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-17 20:53 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-17 21:10 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-17 23:02 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2024-05-20 14:14 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 15:29 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 16:49 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:01 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
@ 2024-05-20 17:22 ` Jacob Champion <[email protected]>
2024-05-20 17:48 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
0 siblings, 1 reply; 46+ messages in thread
From: Jacob Champion @ 2024-05-20 17:22 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; Jacob Burroughs <[email protected]>; PostgreSQL Hackers <[email protected]>
On Mon, May 20, 2024 at 10:01 AM Robert Haas <[email protected]> wrote:
> I really hope that you can't poke big enough holes to kill the feature
> entirely, though. Because that sounds sad.
Even if there are holes, I don't think the situation's going to be bad
enough to tank everything; otherwise no one would be able to use
decompression on the Internet. :D And I expect the authors of the
newer compression methods to have thought about these things [1].
I hesitate to ask as part of the same email, but what were the plans
for compression in combination with transport encryption? (Especially
if you plan to compress the authentication exchange, since mixing your
LDAP password into the compression context seems like it might be a
bad idea if you don't want to leak it.)
--Jacob
[1] https://datatracker.ietf.org/doc/html/rfc8878#name-security-considerations
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-17 20:53 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-17 21:10 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-17 23:02 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2024-05-20 14:14 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 15:29 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 16:49 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:01 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 17:22 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
@ 2024-05-20 17:48 ` Robert Haas <[email protected]>
2024-05-20 18:05 ` Re: libpq compression (part 3) Andrey M. Borodin <[email protected]>
0 siblings, 1 reply; 46+ messages in thread
From: Robert Haas @ 2024-05-20 17:48 UTC (permalink / raw)
To: Jacob Champion <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; Jacob Burroughs <[email protected]>; PostgreSQL Hackers <[email protected]>
On Mon, May 20, 2024 at 1:23 PM Jacob Champion
<[email protected]> wrote:
> On Mon, May 20, 2024 at 10:01 AM Robert Haas <[email protected]> wrote:
> > I really hope that you can't poke big enough holes to kill the feature
> > entirely, though. Because that sounds sad.
>
> Even if there are holes, I don't think the situation's going to be bad
> enough to tank everything; otherwise no one would be able to use
> decompression on the Internet. :D And I expect the authors of the
> newer compression methods to have thought about these things [1].
>
> I hesitate to ask as part of the same email, but what were the plans
> for compression in combination with transport encryption? (Especially
> if you plan to compress the authentication exchange, since mixing your
> LDAP password into the compression context seems like it might be a
> bad idea if you don't want to leak it.)
So, the data would be compressed first, with framing around that, and
then transport encryption would happen afterwards. I don't see how
that would leak your password, but I have a feeling that might be a
sign that I'm about to learn some unpleasant truths.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-17 20:53 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-17 21:10 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-17 23:02 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2024-05-20 14:14 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 15:29 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 16:49 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:01 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 17:22 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:48 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
@ 2024-05-20 18:05 ` Andrey M. Borodin <[email protected]>
2024-05-20 18:37 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
0 siblings, 1 reply; 46+ messages in thread
From: Andrey M. Borodin @ 2024-05-20 18:05 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Jacob Champion <[email protected]>; Jelte Fennema-Nio <[email protected]>; Jacob Burroughs <[email protected]>; PostgreSQL Hackers <[email protected]>
> On 20 May 2024, at 22:48, Robert Haas <[email protected]> wrote:
>
> On Mon, May 20, 2024 at 1:23 PM Jacob Champion
> <[email protected]> wrote:
>> On Mon, May 20, 2024 at 10:01 AM Robert Haas <[email protected]> wrote:
>>> I really hope that you can't poke big enough holes to kill the feature
>>> entirely, though. Because that sounds sad.
>>
>> Even if there are holes, I don't think the situation's going to be bad
>> enough to tank everything; otherwise no one would be able to use
>> decompression on the Internet. :D And I expect the authors of the
>> newer compression methods to have thought about these things [1].
>>
>> I hesitate to ask as part of the same email, but what were the plans
>> for compression in combination with transport encryption? (Especially
>> if you plan to compress the authentication exchange, since mixing your
>> LDAP password into the compression context seems like it might be a
>> bad idea if you don't want to leak it.)
>
> So, the data would be compressed first, with framing around that, and
> then transport encryption would happen afterwards. I don't see how
> that would leak your password, but I have a feeling that might be a
> sign that I'm about to learn some unpleasant truths.
Compression defeats encryption. That's why it's not in TLS anymore.
The thing is compression codecs use data self correlation. And if you mix secret data with user's data, user might guess how correlated they are.
Best regards, Andrey Borodin.
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-17 20:53 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-17 21:10 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-17 23:02 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2024-05-20 14:14 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 15:29 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 16:49 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:01 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 17:22 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:48 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 18:05 ` Re: libpq compression (part 3) Andrey M. Borodin <[email protected]>
@ 2024-05-20 18:37 ` Robert Haas <[email protected]>
2024-05-20 19:09 ` Re: libpq compression (part 3) Andrey M. Borodin <[email protected]>
2024-05-20 19:42 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
0 siblings, 2 replies; 46+ messages in thread
From: Robert Haas @ 2024-05-20 18:37 UTC (permalink / raw)
To: Andrey M. Borodin <[email protected]>; +Cc: Jacob Champion <[email protected]>; Jelte Fennema-Nio <[email protected]>; Jacob Burroughs <[email protected]>; PostgreSQL Hackers <[email protected]>
On Mon, May 20, 2024 at 2:05 PM Andrey M. Borodin <[email protected]> wrote:
> Compression defeats encryption. That's why it's not in TLS anymore.
> The thing is compression codecs use data self correlation. And if you mix secret data with user's data, user might guess how correlated they are.
Yeah, I'm aware that there are some problems like this. For example,
suppose the bad guy can both supply some of the data sent over the
connection (e.g. by typing search queries into a web page) and also
observe the traffic between the web application and the database. Then
they could supply data and try to guess how correlated that is with
other data sent over the same connection. But if that's a practical
attack, preventing compression prior to the authentication exchange
probably isn't good enough: the user could also try to guess what
queries are being sent on behalf of other users through the same
pooled connection, or they could try to use the bits of the query that
they can control to guess what the other bits of the query that they
can't see look like.
But, does this mean that we should just refuse to offer compression as
a feature? This kind of attack isn't a threat in every environment,
and in some environments, compression could be pretty useful. For
instance, you might need to pull down a lot of data from the database
over a slow connection. Perhaps you're the only user of the database,
and you wrote all of the queries yourself in a locked vault, accepting
no untrusted inputs. In that case, these kinds of attacks aren't
possible, or at least I don't see how, but you might want both
compression and encryption. I guess I don't understand why TLS removed
support for encryption entirely instead of disclaiming its use in some
appropriate way.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-17 20:53 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-17 21:10 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-17 23:02 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2024-05-20 14:14 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 15:29 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 16:49 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:01 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 17:22 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:48 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 18:05 ` Re: libpq compression (part 3) Andrey M. Borodin <[email protected]>
2024-05-20 18:37 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
@ 2024-05-20 19:09 ` Andrey M. Borodin <[email protected]>
2024-05-20 20:11 ` Re: libpq compression (part 3) Magnus Hagander <[email protected]>
1 sibling, 1 reply; 46+ messages in thread
From: Andrey M. Borodin @ 2024-05-20 19:09 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Jacob Champion <[email protected]>; Jelte Fennema-Nio <[email protected]>; Jacob Burroughs <[email protected]>; PostgreSQL Hackers <[email protected]>
> On 20 May 2024, at 23:37, Robert Haas <[email protected]> wrote:
>
> But if that's a practical
> attack, preventing compression prior to the authentication exchange
> probably isn't good enough: the user could also try to guess what
> queries are being sent on behalf of other users through the same
> pooled connection, or they could try to use the bits of the query that
> they can control to guess what the other bits of the query that they
> can't see look like.
All these attacks can be practically exploited in a controlled environment.
That's why previous incarnation of this patchset [0] contained a way to reset compression context. And Odyssey AFAIR did it (Dan, coauthor of that patch, implemented the compression in Odyssey).
But attacking authentication is much more straightforward and viable.
> On 20 May 2024, at 23:37, Robert Haas <[email protected]> wrote:
>
> But, does this mean that we should just refuse to offer compression as
> a feature?
No, absolutely, we need the feature.
> I guess I don't understand why TLS removed
> support for encryption entirely instead of disclaiming its use in some
> appropriate way.
I think, the scope of TLS is too broad. HTTPS in turn has a compression. But AFAIK it never compress headers.
IMO we should try to avoid compressing authentication information.
Best regards, Andrey Borodin.
[0] https://commitfest.postgresql.org/38/3499/
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-17 20:53 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-17 21:10 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-17 23:02 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2024-05-20 14:14 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 15:29 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 16:49 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:01 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 17:22 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:48 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 18:05 ` Re: libpq compression (part 3) Andrey M. Borodin <[email protected]>
2024-05-20 18:37 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 19:09 ` Re: libpq compression (part 3) Andrey M. Borodin <[email protected]>
@ 2024-05-20 20:11 ` Magnus Hagander <[email protected]>
2024-05-21 12:32 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
0 siblings, 1 reply; 46+ messages in thread
From: Magnus Hagander @ 2024-05-20 20:11 UTC (permalink / raw)
To: Andrey M. Borodin <[email protected]>; +Cc: Robert Haas <[email protected]>; Jacob Champion <[email protected]>; Jelte Fennema-Nio <[email protected]>; Jacob Burroughs <[email protected]>; PostgreSQL Hackers <[email protected]>
On Mon, May 20, 2024 at 9:09 PM Andrey M. Borodin <[email protected]>
wrote:
>
>
>
> > On 20 May 2024, at 23:37, Robert Haas <[email protected]> wrote:
> >
> > But, does this mean that we should just refuse to offer compression as
> > a feature?
>
> No, absolutely, we need the feature.
>
> > I guess I don't understand why TLS removed
> > support for encryption entirely instead of disclaiming its use in some
> > appropriate way.
>
> I think, the scope of TLS is too broad. HTTPS in turn has a compression.
> But AFAIK it never compress headers.
> IMO we should try to avoid compressing authentication information.
>
That used to be the case in HTTP/1. But header compression was one of the
headline features of HTTP/2, which isn't exactly new anymore. But there's a
special algorithm, HPACK, for it. And then http/3 uses QPACK.
Cloudflare has a pretty decent blog post explaining why and how:
https://blog.cloudflare.com/hpack-the-silent-killer-feature-of-http-2/, or
rfc7541 for all the details.
tl;dr; is yes, let's be careful not to expose headers to a CRIME-style
attack. And I doubt our connections has as much to gain by compressing
"header style" fields as http, so we are probably better off just not
compressing those parts.
--
Magnus Hagander
Me: https://www.hagander.net/ <http://www.hagander.net/;
Work: https://www.redpill-linpro.com/ <http://www.redpill-linpro.com/;
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-17 20:53 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-17 21:10 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-17 23:02 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2024-05-20 14:14 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 15:29 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 16:49 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:01 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 17:22 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:48 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 18:05 ` Re: libpq compression (part 3) Andrey M. Borodin <[email protected]>
2024-05-20 18:37 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 19:09 ` Re: libpq compression (part 3) Andrey M. Borodin <[email protected]>
2024-05-20 20:11 ` Re: libpq compression (part 3) Magnus Hagander <[email protected]>
@ 2024-05-21 12:32 ` Robert Haas <[email protected]>
0 siblings, 0 replies; 46+ messages in thread
From: Robert Haas @ 2024-05-21 12:32 UTC (permalink / raw)
To: Magnus Hagander <[email protected]>; +Cc: Andrey M. Borodin <[email protected]>; Jacob Champion <[email protected]>; Jelte Fennema-Nio <[email protected]>; Jacob Burroughs <[email protected]>; PostgreSQL Hackers <[email protected]>
On Mon, May 20, 2024 at 4:12 PM Magnus Hagander <[email protected]> wrote:
> That used to be the case in HTTP/1. But header compression was one of the headline features of HTTP/2, which isn't exactly new anymore. But there's a special algorithm, HPACK, for it. And then http/3 uses QPACK. Cloudflare has a pretty decent blog post explaining why and how: https://blog.cloudflare.com/hpack-the-silent-killer-feature-of-http-2/, or rfc7541 for all the details.
>
> tl;dr; is yes, let's be careful not to expose headers to a CRIME-style attack. And I doubt our connections has as much to gain by compressing "header style" fields as http, so we are probably better off just not compressing > Work: https://www.redpill-linpro.com/
What do you think constitutes a header in the context of the
PostgreSQL wire protocol?
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-17 20:53 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-17 21:10 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-17 23:02 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2024-05-20 14:14 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 15:29 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 16:49 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:01 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 17:22 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:48 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 18:05 ` Re: libpq compression (part 3) Andrey M. Borodin <[email protected]>
2024-05-20 18:37 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
@ 2024-05-20 19:42 ` Jacob Champion <[email protected]>
2024-05-21 15:23 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-21 15:42 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
1 sibling, 2 replies; 46+ messages in thread
From: Jacob Champion @ 2024-05-20 19:42 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Andrey M. Borodin <[email protected]>; Jelte Fennema-Nio <[email protected]>; Jacob Burroughs <[email protected]>; PostgreSQL Hackers <[email protected]>
On Mon, May 20, 2024 at 11:37 AM Robert Haas <[email protected]> wrote:
> But if that's a practical
> attack, preventing compression prior to the authentication exchange
> probably isn't good enough
I mean... you said it, not me. I'm trying not to rain on the parade
too much, because compression is clearly very valuable. But it makes
me really uncomfortable that we're reintroducing the compression
oracle (especially over the authentication exchange, which is
generally more secret than the rest of the traffic).
> But, does this mean that we should just refuse to offer compression as
> a feature? This kind of attack isn't a threat in every environment,
> and in some environments, compression could be pretty useful. For
> instance, you might need to pull down a lot of data from the database
> over a slow connection. Perhaps you're the only user of the database,
> and you wrote all of the queries yourself in a locked vault, accepting
> no untrusted inputs. In that case, these kinds of attacks aren't
> possible, or at least I don't see how, but you might want both
> compression and encryption.
Right, I think it's reasonable to let a sufficiently
determined/informed user lift the guardrails, but first we have to
choose to put guardrails in place... and then we have to somehow
sufficiently inform the users when it's okay to lift them.
> I guess I don't understand why TLS removed
> support for encryption entirely instead of disclaiming its use in some
> appropriate way.
One of the IETF conversations was at [1] (there were dissenters on the
list, as you might expect). My favorite summary is this one from
Alyssa Rowan:
> Compression is usually best performed as "high" as possible; transport layer is blind to what's being compressed, which is (as we now know) was definitely too low and was in retrospect a mistake.
>
> Any application layer protocol needs to know - if compression is supported - to separate compression contexts for attacker-chosen plaintext and attacker-sought unknown secrets. (As others have stated, HTTPbis covers this.)
But for SQL, where's the dividing line between attacker-chosen and
attacker-sought? To me, it seems like only the user knows; the server
has no clue. I think that puts us "lower" in Alyssa's model than HTTP
is.
As Andrey points out, there was prior work done that started to take
this into account. I haven't reviewed it to see how good it is -- and
I think there are probably many use cases in which queries and tables
contain both private and attacker-controlled information -- but if we
agree that they have to be separated, then the strategy can at least
be improved upon.
--Jacob
[1] https://mailarchive.ietf.org/arch/msg/tls/xhMLf8j4pq8W_ZGXUUU1G_m6r1c/
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-17 20:53 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-17 21:10 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-17 23:02 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2024-05-20 14:14 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 15:29 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 16:49 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:01 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 17:22 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:48 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 18:05 ` Re: libpq compression (part 3) Andrey M. Borodin <[email protected]>
2024-05-20 18:37 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 19:42 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
@ 2024-05-21 15:23 ` Jacob Burroughs <[email protected]>
2024-05-21 18:23 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
1 sibling, 1 reply; 46+ messages in thread
From: Jacob Burroughs @ 2024-05-21 15:23 UTC (permalink / raw)
To: Jacob Champion <[email protected]>; +Cc: Robert Haas <[email protected]>; Andrey M. Borodin <[email protected]>; Jelte Fennema-Nio <[email protected]>; PostgreSQL Hackers <[email protected]>
On Mon, May 20, 2024 at 2:42 PM Jacob Champion
<[email protected]> wrote:
>
> I mean... you said it, not me. I'm trying not to rain on the parade
> too much, because compression is clearly very valuable. But it makes
> me really uncomfortable that we're reintroducing the compression
> oracle (especially over the authentication exchange, which is
> generally more secret than the rest of the traffic).
As currently implemented, the compression only applies to
CopyData/DataRow/Query messages, none of which should be involved in
authentication, unless I've really missed something in my
understanding.
> Right, I think it's reasonable to let a sufficiently
> determined/informed user lift the guardrails, but first we have to
> choose to put guardrails in place... and then we have to somehow
> sufficiently inform the users when it's okay to lift them.
My thought would be that compression should be opt-in on the client
side, with documentation around the potential security pitfalls. (I
could be convinced it should be opt-in on the server side, but overall
I think opt-in on the client side generally protects against footguns
without excessively getting in the way and if an attacker controls the
client, they can just get the information they want directly-they
don't need compression sidechannels to get that information.)
> But for SQL, where's the dividing line between attacker-chosen and
> attacker-sought? To me, it seems like only the user knows; the server
> has no clue. I think that puts us "lower" in Alyssa's model than HTTP
> is.
>
> As Andrey points out, there was prior work done that started to take
> this into account. I haven't reviewed it to see how good it is -- and
> I think there are probably many use cases in which queries and tables
> contain both private and attacker-controlled information -- but if we
> agree that they have to be separated, then the strategy can at least
> be improved upon.
Within SQL-level things, I don't think we can reasonably differentiate
between private and attacker-controlled information at the
libpq/server level. We can reasonably differentiate between message
types that *definitely* are private and ones that could have
either/both data in them, but that's not nearly as useful. I think
not compressing auth-related packets plus giving a mechanism to reset
the compression stream for clients (plus guidance on the tradeoffs
involved in turning on compression) is about as good as we can get.
That said, I *think* the feature is reasonable to be
reviewed/committed without the reset functionality as long as the
compressed data already has the mechanism built in (as it does) to
signal when a decompressor should restart its streaming. The actual
signaling protocol mechanism/necessary libpq API can happen in
followon work.
--
Jacob Burroughs | Staff Software Engineer
E: [email protected]
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-17 20:53 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-17 21:10 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-17 23:02 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2024-05-20 14:14 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 15:29 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 16:49 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:01 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 17:22 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:48 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 18:05 ` Re: libpq compression (part 3) Andrey M. Borodin <[email protected]>
2024-05-20 18:37 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 19:42 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-21 15:23 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
@ 2024-05-21 18:23 ` Jacob Champion <[email protected]>
2024-05-21 19:26 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
0 siblings, 1 reply; 46+ messages in thread
From: Jacob Champion @ 2024-05-21 18:23 UTC (permalink / raw)
To: Jacob Burroughs <[email protected]>; +Cc: Robert Haas <[email protected]>; Andrey M. Borodin <[email protected]>; Jelte Fennema-Nio <[email protected]>; PostgreSQL Hackers <[email protected]>
On Tue, May 21, 2024 at 8:23 AM Jacob Burroughs
<[email protected]> wrote:
> As currently implemented, the compression only applies to
> CopyData/DataRow/Query messages, none of which should be involved in
> authentication, unless I've really missed something in my
> understanding.
Right, but Robert has argued that we should compress it all, and I'm
responding to that proposal.
Sorry for introducing threads within threads. But I think it's
valuable to pin down both 1) the desired behavior, and 2) how the
current proposal behaves, as two separate things. I'll try to do a
better job of communicating which I'm talking about.
> > Right, I think it's reasonable to let a sufficiently
> > determined/informed user lift the guardrails, but first we have to
> > choose to put guardrails in place... and then we have to somehow
> > sufficiently inform the users when it's okay to lift them.
>
> My thought would be that compression should be opt-in on the client
> side, with documentation around the potential security pitfalls. (I
> could be convinced it should be opt-in on the server side, but overall
> I think opt-in on the client side generally protects against footguns
> without excessively getting in the way
We absolutely have to document the risks and allow clients to be
written safely. But I think server-side controls on risky behavior
have proven to be generally more valuable, because the server
administrator is often in a better spot to see the overall risks to
the system. ("No, you will not use deprecated ciphersuites. No, you
will not access this URL over plaintext. No, I will not compress this
response containing customer credit card numbers, no matter how nicely
you ask.") There are many more clients than servers, so it's less
risky for the server to enforce safety than to hope that every client
is safe.
Does your database and access pattern regularly mingle secrets with
public data? Would auditing correct client use of compression be a
logistical nightmare? Do your app developers keep indicating in
conversations that they don't understand the risks at all? Cool, just
set `encrypted_compression = nope_nope_nope` on the server and sleep
soundly at night. (Ideally we would default to that.)
> and if an attacker controls the
> client, they can just get the information they want directly-they
> don't need compression sidechannels to get that information.)
Sure, but I don't think that's relevant to the threats being discussed.
> Within SQL-level things, I don't think we can reasonably differentiate
> between private and attacker-controlled information at the
> libpq/server level.
And by the IETF line of argument -- or at least the argument I quoted
above -- that implies that we really have no business introducing
compression when confidentiality is requested. A stronger approach
would require us to prove, or the user to indicate, safety before
compressing.
Take a look at the security notes for QPACK [1] -- keeping in mind
that they know _more_ about what's going on at the protocol level than
we do, due to the header design. And they still say things like "an
encoder might choose not to index values with low entropy" and "these
criteria ... will evolve over time as new attacks are discovered." A
huge amount is left as an exercise for the reader. This stuff is
really hard.
> We can reasonably differentiate between message
> types that *definitely* are private and ones that could have
> either/both data in them, but that's not nearly as useful. I think
> not compressing auth-related packets plus giving a mechanism to reset
> the compression stream for clients (plus guidance on the tradeoffs
> involved in turning on compression) is about as good as we can get.
The concept of stream reset seems necessary but insufficient at the
application level, which bleeds over into Jelte's compression_restart
proposal. (At the protocol level, I think it may be sufficient?)
If I write a query where one of the WHERE clauses is
attacker-controlled and the other is a secret, I would really like to
not compress that query on the client side. If I join a table of user
IDs against a table of user-provided addresses and a table of
application tokens for that user, compressing even a single row leaks
information about those tokens -- at a _very_ granular level -- and I
would really like the server not to do that.
So if I'm building sand castles... I think maybe it'd be nice to mark
tables (and/or individual columns?) as safe for compression under
encryption, whether by row or in aggregate. And maybe libpq and psql
should be able to turn outgoing compression on and off at will.
And I understand those would balloon the scope of the feature. I'm
worried I'm doing the security-person thing and sucking all the air
out of the room. I know not everybody uses transport encryption; for
those people, compress-it-all is probably a pretty winning strategy,
and there's no need to reset the compression context ever. And the
pg_dump-style, "give me everything" use case seems like it could maybe
be okay, but I really don't know how to assess the risk there, at all.
> That said, I *think* the feature is reasonable to be
> reviewed/committed without the reset functionality as long as the
> compressed data already has the mechanism built in (as it does) to
> signal when a decompressor should restart its streaming. The actual
> signaling protocol mechanism/necessary libpq API can happen in
> followon work.
Well... working out the security minutiae _after_ changing the
protocol is not historically a winning strategy, I think. Better to do
it as a vertical stack.
Thanks,
--Jacob
[1] https://www.rfc-editor.org/rfc/rfc9204.html#name-security-considerations
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-17 20:53 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-17 21:10 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-17 23:02 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2024-05-20 14:14 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 15:29 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 16:49 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:01 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 17:22 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:48 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 18:05 ` Re: libpq compression (part 3) Andrey M. Borodin <[email protected]>
2024-05-20 18:37 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 19:42 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-21 15:23 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-21 18:23 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
@ 2024-05-21 19:26 ` Jacob Burroughs <[email protected]>
0 siblings, 0 replies; 46+ messages in thread
From: Jacob Burroughs @ 2024-05-21 19:26 UTC (permalink / raw)
To: Jacob Champion <[email protected]>; +Cc: Robert Haas <[email protected]>; Andrey M. Borodin <[email protected]>; Jelte Fennema-Nio <[email protected]>; PostgreSQL Hackers <[email protected]>
On Tue, May 21, 2024 at 1:24 PM Jacob Champion
<[email protected]> wrote:
>
> We absolutely have to document the risks and allow clients to be
> written safely. But I think server-side controls on risky behavior
> have proven to be generally more valuable, because the server
> administrator is often in a better spot to see the overall risks to
> the system. ("No, you will not use deprecated ciphersuites. No, you
> will not access this URL over plaintext. No, I will not compress this
> response containing customer credit card numbers, no matter how nicely
> you ask.") There are many more clients than servers, so it's less
> risky for the server to enforce safety than to hope that every client
> is safe.
>
> Does your database and access pattern regularly mingle secrets with
> public data? Would auditing correct client use of compression be a
> logistical nightmare? Do your app developers keep indicating in
> conversations that they don't understand the risks at all? Cool, just
> set `encrypted_compression = nope_nope_nope` on the server and sleep
> soundly at night. (Ideally we would default to that.)
Thinking about this more (and adding a encrypted_compression GUC or
whatever), I think my inclination would on the server-side default
enable compression for insecure connections but default disable for
encrypted connections, but both would be config parameters that can be
changed as desired.
> The concept of stream reset seems necessary but insufficient at the
> application level, which bleeds over into Jelte's compression_restart
> proposal. (At the protocol level, I think it may be sufficient?)
>
> If I write a query where one of the WHERE clauses is
> attacker-controlled and the other is a secret, I would really like to
> not compress that query on the client side. If I join a table of user
> IDs against a table of user-provided addresses and a table of
> application tokens for that user, compressing even a single row leaks
> information about those tokens -- at a _very_ granular level -- and I
> would really like the server not to do that.
>
> So if I'm building sand castles... I think maybe it'd be nice to mark
> tables (and/or individual columns?) as safe for compression under
> encryption, whether by row or in aggregate. And maybe libpq and psql
> should be able to turn outgoing compression on and off at will.
>
> And I understand those would balloon the scope of the feature. I'm
> worried I'm doing the security-person thing and sucking all the air
> out of the room. I know not everybody uses transport encryption; for
> those people, compress-it-all is probably a pretty winning strategy,
> and there's no need to reset the compression context ever. And the
> pg_dump-style, "give me everything" use case seems like it could maybe
> be okay, but I really don't know how to assess the risk there, at all.
I would imagine that a large volume of uses of postgres are in
contexts (e.g. internal networks) where either no encryption is used
or even when encryption is used the benefit of compression vs the risk
of someone being a position to perform a BREACH-style sidechannel
attack against DB traffic is sufficiently high that compress-it-all
would be be quite useful in many cases. Would some sort of
per-table/column marking be useful for some cases? Probably, but that
doesn't seem to me like it needs to be in v1 of this feature as long
as the protocol layer itself is designed such that parties can
arbitrarily alternate between transmitting compressed and uncompressed
data. Then if we build such a feature down the road we just add logic
around *when* we compress but the protocol layer doesn't change.
> Well... working out the security minutiae _after_ changing the
> protocol is not historically a winning strategy, I think. Better to do
> it as a vertical stack.
Thinking about it more, I agree that we probably should work out the
protocol level mechanism for resetting compression
context/enabling/disabling/reconfiguring compression as part of this
work. I don't think that we need to have all the ways that the
application layer might choose to use such things done here, but we
should have all the necessary primitives worked out.
--
Jacob Burroughs | Staff Software Engineer
E: [email protected]
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-17 20:53 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-17 21:10 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-17 23:02 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2024-05-20 14:14 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 15:29 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 16:49 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:01 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 17:22 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:48 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 18:05 ` Re: libpq compression (part 3) Andrey M. Borodin <[email protected]>
2024-05-20 18:37 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 19:42 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
@ 2024-05-21 15:42 ` Jelte Fennema-Nio <[email protected]>
2024-05-21 16:13 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
1 sibling, 1 reply; 46+ messages in thread
From: Jelte Fennema-Nio @ 2024-05-21 15:42 UTC (permalink / raw)
To: Jacob Champion <[email protected]>; +Cc: Robert Haas <[email protected]>; Andrey M. Borodin <[email protected]>; Jacob Burroughs <[email protected]>; PostgreSQL Hackers <[email protected]>
On Mon, 20 May 2024 at 21:42, Jacob Champion
<[email protected]> wrote:
> As Andrey points out, there was prior work done that started to take
> this into account. I haven't reviewed it to see how good it is -- and
> I think there are probably many use cases in which queries and tables
> contain both private and attacker-controlled information -- but if we
> agree that they have to be separated, then the strategy can at least
> be improved upon.
To help get everyone on the same page I wanted to list all the
security concerns in one place:
1. Triggering excessive CPU usage before authentication, by asking for
very high compression levels
2. Triggering excessive memory/CPU usage before authentication, by
sending a client sending a zipbomb
3. Triggering excessive CPU after authentication, by asking for a very
high compression level
4. Triggering excessive memory/CPU after authentication due to
zipbombs (i.e. small amount of data extracting to lots of data)
5. CRIME style leakage of information about encrypted data
1 & 2 can easily be solved by not allowing any authentication packets
to be compressed. This also has benefits for 5.
3 & 4 are less of a concern than 1&2 imho. Once authenticated a client
deserves some level of trust. But having knobs to limit impact
definitely seems useful.
3 can be solved in two ways afaict:
a. Allow the server to choose the maximum compression level for each
compression method (using some GUC), and downgrade the level
transparently when a higher level is requested
b. Don't allow the client to choose the compression level that the server uses.
I'd prefer option a
4 would require some safety limits on the amount of data that a
(small) compressed message can be decompressed to, and stop
decompression of that message once that limit is hit. What that limit
should be seems hard to choose though. A few ideas:
a. The size of the message reported by the uncompressed header. This
would mean that at most the 4GB will be uncompressed, since maximum
message length is 4GB (limited by 32bit message length field)
b. Allow servers to specify maximum client decompressed message length
lower than this 4GB, e.g. messages of more than 100MB of uncompressed
size should not be allowed.
I think 5 is the most complicated to deal with, especially as it
depends on the actual usage to know what is safe. I believe we should
let users have the freedom to make their own security tradeoffs, but
we should protect them against some of the most glaring issues
(especially ones that benefit little from compression anyway). As
already shown by Andrey, sending LDAP passwords in a compressed way
seems extremely dangerous. So I think we should disallow compressing
any authentication related packets. To reduce similar risks further we
can choose to compress only the message types that we expect to
benefit most from compression. IMHO those are the following (marked
with (B)ackend or (F)rontend to show who sends them):
- Query (F)
- Parse (F)
- Describe (F)
- Bind (F)
- RowDescription (B)
- DataRow (B)
- CopyData (B/F)
Then I think we should let users choose how they want to compress and
where they want their compression stream to restart. Something like
this:
a. compression_restart=query: Restart the stream after every query.
Recommended if queries across the same connection are triggered by
different end-users. I think this would be a sane default
b. compression_restart=message: Restart the stream for every message.
Recommended if the amount of correlation between rows of the same
query is a security concern.
c. compression_restart=manual: Don't restart the stream automatically,
but only when the client user calls a specific function. Recommended
only if the user can make trade-offs, or if no encryption is used
anyway.
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-17 20:53 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-17 21:10 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-17 23:02 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2024-05-20 14:14 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 15:29 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 16:49 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:01 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 17:22 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:48 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 18:05 ` Re: libpq compression (part 3) Andrey M. Borodin <[email protected]>
2024-05-20 18:37 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 19:42 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-21 15:42 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
@ 2024-05-21 16:13 ` Jacob Burroughs <[email protected]>
2024-05-21 18:38 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
0 siblings, 1 reply; 46+ messages in thread
From: Jacob Burroughs @ 2024-05-21 16:13 UTC (permalink / raw)
To: Jelte Fennema-Nio <[email protected]>; +Cc: Jacob Champion <[email protected]>; Robert Haas <[email protected]>; Andrey M. Borodin <[email protected]>; PostgreSQL Hackers <[email protected]>
On Tue, May 21, 2024 at 10:43 AM Jelte Fennema-Nio <[email protected]> wrote:
>
> To help get everyone on the same page I wanted to list all the
> security concerns in one place:
>
> 1. Triggering excessive CPU usage before authentication, by asking for
> very high compression levels
> 2. Triggering excessive memory/CPU usage before authentication, by
> sending a client sending a zipbomb
> 3. Triggering excessive CPU after authentication, by asking for a very
> high compression level
> 4. Triggering excessive memory/CPU after authentication due to
> zipbombs (i.e. small amount of data extracting to lots of data)
> 5. CRIME style leakage of information about encrypted data
>
> 1 & 2 can easily be solved by not allowing any authentication packets
> to be compressed. This also has benefits for 5.
This is already addressed by only compressing certain message types.
If we think it is important that the server reject compressed packets
of other types I can add that, but it seemed reasonable to just make
the client never send such packets compressed.
> 3 & 4 are less of a concern than 1&2 imho. Once authenticated a client
> deserves some level of trust. But having knobs to limit impact
> definitely seems useful.
>
> 3 can be solved in two ways afaict:
> a. Allow the server to choose the maximum compression level for each
> compression method (using some GUC), and downgrade the level
> transparently when a higher level is requested
> b. Don't allow the client to choose the compression level that the server uses.
>
> I'd prefer option a
3a would seem preferable given discussion upthread. It would probably
be worth doing some measurement to check how much of an actual
difference in compute effort the max vs the default for all 3
algorithms adds, because I would really prefer to avoid needing to add
even more configuration knobs if the max compression level for the
streaming data usecase is sufficiently performant.
> 4 would require some safety limits on the amount of data that a
> (small) compressed message can be decompressed to, and stop
> decompression of that message once that limit is hit. What that limit
> should be seems hard to choose though. A few ideas:
> a. The size of the message reported by the uncompressed header. This
> would mean that at most the 4GB will be uncompressed, since maximum
> message length is 4GB (limited by 32bit message length field)
> b. Allow servers to specify maximum client decompressed message length
> lower than this 4GB, e.g. messages of more than 100MB of uncompressed
> size should not be allowed.
Because we are using streaming decompression, this is much less of an
issue than for things that decompress wholesale onto disk/into memory.
We only read PQ_RECV_BUFFER_SIZE (8k) bytes off the stream at once,
and when reading a packet we already have a `maxmsglen` that is
PQ_LARGE_MESSAGE_LIMIT (1gb) already, and "We abort the connection (by
returning EOF) if client tries to send more than that.)". Therefore,
we effectively already have a limit of 1gb that applies to regular
messages too, and I think we should rely on this mechanism for
compressed data too (if we really think we need to make that number
configurable we probably could, but again the fewer new knobs we need
to add the better.
> I think 5 is the most complicated to deal with, especially as it
> depends on the actual usage to know what is safe. I believe we should
> let users have the freedom to make their own security tradeoffs, but
> we should protect them against some of the most glaring issues
> (especially ones that benefit little from compression anyway). As
> already shown by Andrey, sending LDAP passwords in a compressed way
> seems extremely dangerous. So I think we should disallow compressing
> any authentication related packets. To reduce similar risks further we
> can choose to compress only the message types that we expect to
> benefit most from compression. IMHO those are the following (marked
> with (B)ackend or (F)rontend to show who sends them):
> - Query (F)
> - Parse (F)
> - Describe (F)
> - Bind (F)
> - RowDescription (B)
> - DataRow (B)
> - CopyData (B/F)
That seems like a reasonable list (current implementation is just
CopyData/DataRow/Query, but I really just copied that fairly blindly
from the previous incarnation of this effort.) See also my comment
below 1&2 for if we think we need to block decompressing them too.
> Then I think we should let users choose how they want to compress and
> where they want their compression stream to restart. Something like
> this:
> a. compression_restart=query: Restart the stream after every query.
> Recommended if queries across the same connection are triggered by
> different end-users. I think this would be a sane default
> b. compression_restart=message: Restart the stream for every message.
> Recommended if the amount of correlation between rows of the same
> query is a security concern.
> c. compression_restart=manual: Don't restart the stream automatically,
> but only when the client user calls a specific function. Recommended
> only if the user can make trade-offs, or if no encryption is used
> anyway.
I reasonably like this idea, though I think maybe we should also
(instead of query?) add per-transaction on the backend side. I'm
curious what other people think of this.
--
Jacob Burroughs | Staff Software Engineer
E: [email protected]
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-17 20:53 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-17 21:10 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-17 23:02 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2024-05-20 14:14 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 15:29 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 16:49 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:01 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 17:22 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:48 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 18:05 ` Re: libpq compression (part 3) Andrey M. Borodin <[email protected]>
2024-05-20 18:37 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 19:42 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-21 15:42 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2024-05-21 16:13 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
@ 2024-05-21 18:38 ` Jacob Champion <[email protected]>
2024-05-21 19:08 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
0 siblings, 1 reply; 46+ messages in thread
From: Jacob Champion @ 2024-05-21 18:38 UTC (permalink / raw)
To: Jacob Burroughs <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; Robert Haas <[email protected]>; Andrey M. Borodin <[email protected]>; PostgreSQL Hackers <[email protected]>
On Tue, May 21, 2024 at 9:14 AM Jacob Burroughs
<[email protected]> wrote:
> On Tue, May 21, 2024 at 10:43 AM Jelte Fennema-Nio <[email protected]> wrote:
> > To help get everyone on the same page I wanted to list all the
> > security concerns in one place:
> >
> > 1. Triggering excessive CPU usage before authentication, by asking for
> > very high compression levels
> > 2. Triggering excessive memory/CPU usage before authentication, by
> > sending a client sending a zipbomb
> > 3. Triggering excessive CPU after authentication, by asking for a very
> > high compression level
> > 4. Triggering excessive memory/CPU after authentication due to
> > zipbombs (i.e. small amount of data extracting to lots of data)
> > 5. CRIME style leakage of information about encrypted data
> >
> > 1 & 2 can easily be solved by not allowing any authentication packets
> > to be compressed. This also has benefits for 5.
>
> This is already addressed by only compressing certain message types.
> If we think it is important that the server reject compressed packets
> of other types I can add that, but it seemed reasonable to just make
> the client never send such packets compressed.
If the server doesn't reject compressed packets pre-authentication,
then case 2 isn't mitigated. (I haven't proven how risky that case is
yet, to be clear.) In other words: if the threat model is that a
client can attack us, we shouldn't assume that it will attack us
politely.
> > 4 would require some safety limits on the amount of data that a
> > (small) compressed message can be decompressed to, and stop
> > decompression of that message once that limit is hit. What that limit
> > should be seems hard to choose though. A few ideas:
> > a. The size of the message reported by the uncompressed header. This
> > would mean that at most the 4GB will be uncompressed, since maximum
> > message length is 4GB (limited by 32bit message length field)
> > b. Allow servers to specify maximum client decompressed message length
> > lower than this 4GB, e.g. messages of more than 100MB of uncompressed
> > size should not be allowed.
>
> Because we are using streaming decompression, this is much less of an
> issue than for things that decompress wholesale onto disk/into memory.
(I agree in general, but since you're designing a protocol extension,
IMO it's not enough that your implementation happens to mitigate
risks. We more or less have to bake those mitigations into the
specification of the extension, because things that aren't servers
have to decompress now. Similar to RFC security considerations.)
--Jacob
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-17 20:53 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-17 21:10 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-17 23:02 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2024-05-20 14:14 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 15:29 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 16:49 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:01 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 17:22 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:48 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 18:05 ` Re: libpq compression (part 3) Andrey M. Borodin <[email protected]>
2024-05-20 18:37 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 19:42 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-21 15:42 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2024-05-21 16:13 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-21 18:38 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
@ 2024-05-21 19:08 ` Jacob Burroughs <[email protected]>
2024-05-21 19:42 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
0 siblings, 1 reply; 46+ messages in thread
From: Jacob Burroughs @ 2024-05-21 19:08 UTC (permalink / raw)
To: Jacob Champion <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; Robert Haas <[email protected]>; Andrey M. Borodin <[email protected]>; PostgreSQL Hackers <[email protected]>
On Tue, May 21, 2024 at 1:39 PM Jacob Champion
<[email protected]> wrote:
>
> If the server doesn't reject compressed packets pre-authentication,
> then case 2 isn't mitigated. (I haven't proven how risky that case is
> yet, to be clear.) In other words: if the threat model is that a
> client can attack us, we shouldn't assume that it will attack us
> politely.
I think I thought I was writing about something else when I wrote that
:sigh:. I think what I really should have written was a version of
the part below, which is that we use streaming decompression, only
decompress 8kb at a time, and for pre-auth messages limit them to
`PG_MAX_AUTH_TOKEN_LENGTH` (65535 bytes), which isn't really enough
data to actually cause any real-world pain by needing to decompress vs
the equivalent pain of sending invalid uncompressed auth packets.
> > Because we are using streaming decompression, this is much less of an
> > issue than for things that decompress wholesale onto disk/into memory.
>
> (I agree in general, but since you're designing a protocol extension,
> IMO it's not enough that your implementation happens to mitigate
> risks. We more or less have to bake those mitigations into the
> specification of the extension, because things that aren't servers
> have to decompress now. Similar to RFC security considerations.)
We own both the canonical client and server, so those are both covered
here. I would think it would be the responsibility of any other
system that maintains its own implementation of the postgres protocol
and chooses to support the compression protocol to perform its own
mitigations against potential compression security issues. Should we
put the fixed message size limits (that have de facto been part of the
protocol since 2021, even if they weren't documented as such) into the
protocol documentation? That would give implementers of the protocol
numbers that they could actually rely on when implementing the
appropriate safeguards because they would be able to actually have
explicit guarantees about the size of messages. I think it would make
more sense to put the limits on the underlying messages rather than
adding an additional limit that only applies to compressed messages.
( I don't really see how one could implement other tooling that used
pg compression without using streaming compression, as the protocol
never hands over a standalone blob of compressed data: all compressed
data is always part of a stream, but even with streaming decompression
you still need some kind of limits or you will just chew up memory.)
--
Jacob Burroughs | Staff Software Engineer
E: [email protected]
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-17 20:53 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-17 21:10 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-17 23:02 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2024-05-20 14:14 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 15:29 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 16:49 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:01 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 17:22 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:48 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 18:05 ` Re: libpq compression (part 3) Andrey M. Borodin <[email protected]>
2024-05-20 18:37 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 19:42 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-21 15:42 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2024-05-21 16:13 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-21 18:38 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-21 19:08 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
@ 2024-05-21 19:42 ` Jacob Champion <[email protected]>
0 siblings, 0 replies; 46+ messages in thread
From: Jacob Champion @ 2024-05-21 19:42 UTC (permalink / raw)
To: Jacob Burroughs <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; Robert Haas <[email protected]>; Andrey M. Borodin <[email protected]>; PostgreSQL Hackers <[email protected]>
On Tue, May 21, 2024 at 12:08 PM Jacob Burroughs
<[email protected]> wrote:
> I think I thought I was writing about something else when I wrote that
> :sigh:. I think what I really should have written was a version of
> the part below, which is that we use streaming decompression, only
> decompress 8kb at a time, and for pre-auth messages limit them to
> `PG_MAX_AUTH_TOKEN_LENGTH` (65535 bytes), which isn't really enough
> data to actually cause any real-world pain by needing to decompress vs
> the equivalent pain of sending invalid uncompressed auth packets.
Okay. So it sounds like your position is similar to Robert's from
earlier: prefer allowing unauthenticated compressed packets for
simplicity, as long as we think it's safe for the server. (Personally
I still think a client that compresses its password packets is doing
it wrong, and we could help them out by refusing that.)
> We own both the canonical client and server, so those are both covered
> here. I would think it would be the responsibility of any other
> system that maintains its own implementation of the postgres protocol
> and chooses to support the compression protocol to perform its own
> mitigations against potential compression security issues.
Sure, but if our official documentation is "here's an extremely
security-sensitive feature, figure it out!" then we've done a
disservice to the community.
> Should we
> put the fixed message size limits (that have de facto been part of the
> protocol since 2021, even if they weren't documented as such) into the
> protocol documentation?
Possibly? I don't know if the other PG-compatible implementations use
the same limits. It might be better to say "limits must exist".
> ( I don't really see how one could implement other tooling that used
> pg compression without using streaming compression, as the protocol
> never hands over a standalone blob of compressed data: all compressed
> data is always part of a stream, but even with streaming decompression
> you still need some kind of limits or you will just chew up memory.)
Well, that's a good point; I wasn't thinking about the streaming APIs
themselves. If the easiest way to implement decompression requires the
use of an API that shouts "hey, give me guardrails!", then that helps
quite a bit. I really need to look into the attack surface of the
three algorithms.
--Jacob
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-17 20:53 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
@ 2024-05-17 21:40 ` Robert Haas <[email protected]>
2024-05-17 22:54 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
1 sibling, 1 reply; 46+ messages in thread
From: Robert Haas @ 2024-05-17 21:40 UTC (permalink / raw)
To: Jacob Burroughs <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; PostgreSQL Hackers <[email protected]>
On Fri, May 17, 2024 at 4:53 PM Jacob Burroughs
<[email protected]> wrote:
> I think I was more thinking that trying to let both parties control
> the parameter seemed like a recipe for confusion and sadness, and so
> the choice that felt most natural to me was to let the sender control
> it, but I'm definitely open to changing that the other way around.
To be clear, I am not arguing that it should be the receiver's choice.
I'm arguing it should be the client's choice, which means the client
decides what it sends and also tells the server what to send to it.
I'm open to counter-arguments, but as I've thought about this more,
I've come to the conclusion that letting the client control the
behavior is the most likely to be useful and the most consistent with
existing facilities. I think we're on the same page based on the rest
of your email: I'm just clarifying.
> On the server side, we use slash separated sets of options
> connection_compression=DEFAULT_VALUE_FOR_BOTH_DIRECTIONS/client_to_server=OVERRIDE_FOR_THIS_DIRECTION/server_to_client=OVERRIDE_FOR_THIS_DIRECTION
> with the values being semicolon separated compression algorithms.
> On the client side, you can specify
> compression=<same_specification_as_above>,
> but on the client side you can actually specify compression options,
> which the server will use if provided, and otherwise it will fall back
> to defaults.
I have some quibbles with the syntax but I agree with the concept.
What I'd probably do is separate the server side thing into two GUCs,
each with a list of algorithms, comma-separated, like we do for other
lists in postgresql.conf. Maybe make the default 'all' meaning
"everything this build of the server supports". On the client side,
I'd allow both things to be specified using a single option, because
wanting to do the same thing in both directions will be common, and
you actually have to type in connection strings sometimes, so
verbosity matters more.
As far as the format of the value for that keyword, what do you think
about either compression=DO_THIS_BOTH_WAYS or
compression=DO_THIS_WHEN_SENDING/DO_THIS_WHEN_RECEIVING, with each "do
this" being a specification of the same form already accepted for
server-side compression e.g. gzip or gzip:level=9? If you don't like
that, why do you think the proposal you made above is better, and why
is that one now punctuated with slashes instead of semicolons?
> If we think we need to, we could let the server specify defaults for
> server-side compression. My overall thought though is that having an
> excessive number of knobs increases the surface area for testing and
> bugs while also increasing potential user confusion and that allowing
> configuration on *both* sides doesn't seem sufficiently useful to be
> worth adding that complexity.
I agree.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 46+ messages in thread
* Re: libpq compression (part 3)
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-17 20:53 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-17 21:40 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
@ 2024-05-17 22:54 ` Jelte Fennema-Nio <[email protected]>
0 siblings, 0 replies; 46+ messages in thread
From: Jelte Fennema-Nio @ 2024-05-17 22:54 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Jacob Burroughs <[email protected]>; PostgreSQL Hackers <[email protected]>
On Fri, 17 May 2024 at 23:40, Robert Haas <[email protected]> wrote:
> To be clear, I am not arguing that it should be the receiver's choice.
> I'm arguing it should be the client's choice, which means the client
> decides what it sends and also tells the server what to send to it.
> I'm open to counter-arguments, but as I've thought about this more,
> I've come to the conclusion that letting the client control the
> behavior is the most likely to be useful and the most consistent with
> existing facilities. I think we're on the same page based on the rest
> of your email: I'm just clarifying.
+1
> I have some quibbles with the syntax but I agree with the concept.
> What I'd probably do is separate the server side thing into two GUCs,
> each with a list of algorithms, comma-separated, like we do for other
> lists in postgresql.conf. Maybe make the default 'all' meaning
> "everything this build of the server supports". On the client side,
> I'd allow both things to be specified using a single option, because
> wanting to do the same thing in both directions will be common, and
> you actually have to type in connection strings sometimes, so
> verbosity matters more.
>
> As far as the format of the value for that keyword, what do you think
> about either compression=DO_THIS_BOTH_WAYS or
> compression=DO_THIS_WHEN_SENDING/DO_THIS_WHEN_RECEIVING, with each "do
> this" being a specification of the same form already accepted for
> server-side compression e.g. gzip or gzip:level=9? If you don't like
> that, why do you think the proposal you made above is better, and why
> is that one now punctuated with slashes instead of semicolons?
+1
^ permalink raw reply [nested|flat] 46+ messages in thread
end of thread, other threads:[~2024-05-21 19:42 UTC | newest]
Thread overview: 46+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2023-08-17 16:00 [PATCH v7 1/1] Introduce macros for protocol characters. Nathan Bossart <[email protected]>
2023-12-20 21:48 Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2023-12-21 00:30 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2023-12-29 10:02 ` Re: libpq compression (part 3) Andrey M. Borodin <[email protected]>
2023-12-31 07:32 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 20:45 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-01-12 21:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-01-12 21:11 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:08 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 16:30 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 18:35 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 19:22 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-14 20:23 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-14 21:21 ` Fwd: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 13:38 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:24 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-15 16:31 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-15 16:50 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-16 13:28 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-17 20:53 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-17 21:10 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-17 23:02 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2024-05-18 05:18 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-20 15:17 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 14:14 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 15:29 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 16:49 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:01 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 17:22 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-20 17:48 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 18:05 ` Re: libpq compression (part 3) Andrey M. Borodin <[email protected]>
2024-05-20 18:37 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 19:09 ` Re: libpq compression (part 3) Andrey M. Borodin <[email protected]>
2024-05-20 20:11 ` Re: libpq compression (part 3) Magnus Hagander <[email protected]>
2024-05-21 12:32 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-20 19:42 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-21 15:23 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-21 18:23 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-21 19:26 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-21 15:42 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
2024-05-21 16:13 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-21 18:38 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-21 19:08 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]>
2024-05-21 19:42 ` Re: libpq compression (part 3) Jacob Champion <[email protected]>
2024-05-17 21:40 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
2024-05-17 22:54 ` Re: libpq compression (part 3) Jelte Fennema-Nio <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox